diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index 87274a5133d1cf..75346f2c7618c2 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -1250,7 +1250,7 @@ instance:: >>> d.f.__self__ - <__main__.D object at 0x1012e1f98> + <__main__.D object at 0x00B18C90> If you have ever wondered where *self* comes from in regular methods or where *cls* comes from in class methods, this is it! diff --git a/Doc/library/msvcrt.rst b/Doc/library/msvcrt.rst index 0b059e746c61af..2a6d980ab78a60 100644 --- a/Doc/library/msvcrt.rst +++ b/Doc/library/msvcrt.rst @@ -75,10 +75,14 @@ File Operations .. function:: open_osfhandle(handle, flags) Create a C runtime file descriptor from the file handle *handle*. The *flags* - parameter should be a bitwise OR of :const:`os.O_APPEND`, :const:`os.O_RDONLY`, - and :const:`os.O_TEXT`. The returned file descriptor may be used as a parameter + parameter should be a bitwise OR of :const:`os.O_APPEND`, + :const:`os.O_RDONLY`, :const:`os.O_TEXT` and :const:`os.O_NOINHERIT`. + The returned file descriptor may be used as a parameter to :func:`os.fdopen` to create a file object. + The file descriptor is inheritable by default. Pass :const:`os.O_NOINHERIT` + flag to make it non inheritable. + .. audit-event:: msvcrt.open_osfhandle handle,flags msvcrt.open_osfhandle diff --git a/Include/internal/pycore_freelist.h b/Include/internal/pycore_freelist.h index 3c3ba38db0197a..34009435910d99 100644 --- a/Include/internal/pycore_freelist.h +++ b/Include/internal/pycore_freelist.h @@ -61,8 +61,8 @@ struct _Py_float_state { typedef struct _Py_freelist_state { struct _Py_float_state float_state; - struct _Py_list_state list; struct _Py_tuple_state tuple_state; + struct _Py_list_state list_state; } _PyFreeListState; #ifdef __cplusplus diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index 47970438c0881f..c029b239306648 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -122,6 +122,10 @@ static inline void _PyGC_SET_FINALIZED(PyObject *op) { PyGC_Head *gc = _Py_AS_GC(op); _PyGCHead_SET_FINALIZED(gc); } +static inline void _PyGC_CLEAR_FINALIZED(PyObject *op) { + PyGC_Head *gc = _Py_AS_GC(op); + gc->_gc_prev &= ~_PyGC_PREV_MASK_FINALIZED; +} /* GC runtime state */ diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index a49630112af510..7fa0a85100a581 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -1701,6 +1701,14 @@ def test_asend(self): async def gen(): yield 1 + # gh-113753: asend objects allocated from a free-list should warn. + # Ensure there is a finalized 'asend' object ready to be reused. + try: + g = gen() + g.asend(None).send(None) + except StopIteration: + pass + msg = f"coroutine method 'asend' of '{gen.__qualname__}' was never awaited" with self.assertWarnsRegex(RuntimeWarning, msg): g = gen() diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index e15492783aeec1..fcddd147bac63e 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -629,8 +629,8 @@ def __dir__(self): def test___ne__(self): self.assertFalse(None.__ne__(None)) - self.assertTrue(None.__ne__(0)) - self.assertTrue(None.__ne__("abc")) + self.assertIs(None.__ne__(0), NotImplemented) + self.assertIs(None.__ne__("abc"), NotImplemented) def test_divmod(self): self.assertEqual(divmod(12, 7), (1, 5)) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index c66c5797471413..bff6e604cccdd6 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -4485,6 +4485,61 @@ def test_openpty(self): self.assertEqual(os.get_inheritable(master_fd), False) self.assertEqual(os.get_inheritable(slave_fd), False) + @unittest.skipUnless(hasattr(os, 'spawnl'), "need os.openpty()") + def test_pipe_spawnl(self): + # gh-77046: On Windows, os.pipe() file descriptors must be created with + # _O_NOINHERIT to make them non-inheritable. UCRT has no public API to + # get (_osfile(fd) & _O_NOINHERIT), so use a functional test. + # + # Make sure that fd is not inherited by a child process created by + # os.spawnl(): get_osfhandle() and dup() must fail with EBADF. + + fd, fd2 = os.pipe() + self.addCleanup(os.close, fd) + self.addCleanup(os.close, fd2) + + code = textwrap.dedent(f""" + import errno + import os + import test.support + try: + import msvcrt + except ImportError: + msvcrt = None + + fd = {fd} + + with test.support.SuppressCrashReport(): + if msvcrt is not None: + try: + handle = msvcrt.get_osfhandle(fd) + except OSError as exc: + if exc.errno != errno.EBADF: + raise + # get_osfhandle(fd) failed with EBADF as expected + else: + raise Exception("get_osfhandle() must fail") + + try: + fd3 = os.dup(fd) + except OSError as exc: + if exc.errno != errno.EBADF: + raise + # os.dup(fd) failed with EBADF as expected + else: + os.close(fd3) + raise Exception("dup must fail") + """) + + filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + with open(filename, "w") as fp: + print(code, file=fp, end="") + + cmd = [sys.executable, filename] + exitcode = os.spawnl(os.P_WAIT, cmd[0], *cmd) + self.assertEqual(exitcode, 0) + class PathTConverterTests(unittest.TestCase): # tuples of (function name, allows fd arguments, additional arguments to diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 102e697ba7a90d..944a7de4210bc9 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -835,6 +835,11 @@ def is_env_var_to_ignore(n): if not is_env_var_to_ignore(k)] self.assertEqual(child_env_names, []) + @unittest.skipIf(sysconfig.get_config_var('Py_ENABLE_SHARED') == 1, + 'The Python shared library cannot be loaded ' + 'without some system environments.') + @unittest.skipIf(check_sanitizer(address=True), + 'AddressSanitizer adds to the environment.') def test_one_environment_variable(self): newenv = {'fruit': 'orange'} cmd = [sys.executable, '-c', @@ -842,9 +847,13 @@ def test_one_environment_variable(self): 'sys.stdout.write("fruit="+os.getenv("fruit"))'] if sys.platform == "win32": cmd = ["CMD", "/c", "SET", "fruit"] - with subprocess.Popen(cmd, stdout=subprocess.PIPE, env=newenv) as p: - stdout, _ = p.communicate() - self.assertTrue(stdout.startswith(b"fruit=orange")) + with subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=newenv) as p: + stdout, stderr = p.communicate() + if p.returncode and support.verbose: + print("STDOUT:", stdout.decode("ascii", "replace")) + print("STDERR:", stderr.decode("ascii", "replace")) + self.assertEqual(p.returncode, 0) + self.assertEqual(stdout.strip(), b"fruit=orange") def test_invalid_cmd(self): # null character in the command name diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index 41ce81a9d08c4b..f7b6db465b4bc7 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -2272,6 +2272,7 @@ def test_decompress_without_3rd_party_library(self): with zipfile.ZipFile(zip_file) as zf: self.assertRaises(RuntimeError, zf.extract, 'a.txt') + @requires_zlib() def test_full_overlap(self): data = ( b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e' @@ -2300,6 +2301,7 @@ def test_full_overlap(self): with self.assertRaisesRegex(zipfile.BadZipFile, 'File name.*differ'): zipf.read('b') + @requires_zlib() def test_quoted_overlap(self): data = ( b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05Y\xfc' diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-05-21-28-48.gh-issue-113753.2HNiuq.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-05-21-28-48.gh-issue-113753.2HNiuq.rst new file mode 100644 index 00000000000000..32cf2cb2a4ae56 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-05-21-28-48.gh-issue-113753.2HNiuq.rst @@ -0,0 +1,2 @@ +Fix an issue where the finalizer of ``PyAsyncGenASend`` objects might not be +called if they were allocated from a free list. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-01-08-14-34-02.gh-issue-77046.sDUh2d.rst b/Misc/NEWS.d/next/Core and Builtins/2024-01-08-14-34-02.gh-issue-77046.sDUh2d.rst new file mode 100644 index 00000000000000..9f0f144451df6c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-01-08-14-34-02.gh-issue-77046.sDUh2d.rst @@ -0,0 +1,3 @@ +On Windows, file descriptors wrapping Windows handles are now created non +inheritable by default (:pep:`446`). Patch by Zackery Spytz and Victor +Stinner. diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index fecb3389570780..54e15555417287 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -391,9 +391,9 @@ _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, } if (self->writable) - self->fd = _Py_open_osfhandle_noraise(handle, _O_WRONLY | _O_BINARY); + self->fd = _Py_open_osfhandle_noraise(handle, _O_WRONLY | _O_BINARY | _O_NOINHERIT); else - self->fd = _Py_open_osfhandle_noraise(handle, _O_RDONLY | _O_BINARY); + self->fd = _Py_open_osfhandle_noraise(handle, _O_RDONLY | _O_BINARY | _O_NOINHERIT); if (self->fd < 0) { PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, nameobj); CloseHandle(handle); diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 39b1f3cb7b2b9b..179497a21b5a95 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -11578,8 +11578,8 @@ os_pipe_impl(PyObject *module) Py_BEGIN_ALLOW_THREADS ok = CreatePipe(&read, &write, &attr, 0); if (ok) { - fds[0] = _Py_open_osfhandle_noraise(read, _O_RDONLY); - fds[1] = _Py_open_osfhandle_noraise(write, _O_WRONLY); + fds[0] = _Py_open_osfhandle_noraise(read, _O_RDONLY | _O_NOINHERIT); + fds[1] = _Py_open_osfhandle_noraise(write, _O_WRONLY | _O_NOINHERIT); if (fds[0] == -1 || fds[1] == -1) { CloseHandle(read); CloseHandle(write); diff --git a/Objects/genobject.c b/Objects/genobject.c index 9614713883741c..f03919c75d70a5 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -6,6 +6,7 @@ #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_ceval.h" // _PyEval_EvalFrame() #include "pycore_frame.h" // _PyInterpreterFrame +#include "pycore_gc.h" // _PyGC_CLEAR_FINALIZED() #include "pycore_genobject.h" // struct _Py_async_gen_state #include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_object.h" // _PyObject_GC_UNTRACK() @@ -1739,6 +1740,7 @@ async_gen_asend_dealloc(PyAsyncGenASend *o) #endif if (state->asend_numfree < _PyAsyncGen_MAXFREELIST) { assert(PyAsyncGenASend_CheckExact(o)); + _PyGC_CLEAR_FINALIZED((PyObject *)o); state->asend_freelist[state->asend_numfree++] = o; } else diff --git a/Objects/listobject.c b/Objects/listobject.c index 2fc57e13f632f8..c05c4fdff83883 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -26,7 +26,7 @@ get_list_state(void) { _PyFreeListState *state = _PyFreeListState_GET(); assert(state != NULL); - return &state->list; + return &state->list_state; } #endif @@ -124,7 +124,7 @@ void _PyList_ClearFreeList(_PyFreeListState *freelist_state, int is_finalization) { #if PyList_MAXFREELIST > 0 - struct _Py_list_state *state = &freelist_state->list; + struct _Py_list_state *state = &freelist_state->list_state; while (state->numfree > 0) { PyListObject *op = state->free_list[--state->numfree]; assert(PyList_CheckExact(op)); diff --git a/Tools/build/generate_sbom.py b/Tools/build/generate_sbom.py index 93d0d8a3762df3..282ee20cc402b0 100644 --- a/Tools/build/generate_sbom.py +++ b/Tools/build/generate_sbom.py @@ -82,6 +82,14 @@ def spdx_id(value: str) -> str: return re.sub(r"[^a-zA-Z0-9.\-]+", "-", value) +def error_if(value: bool, error_message: str) -> None: + """Prints an error if a comparison fails along with a link to the devguide""" + if value: + print(error_message) + print("See 'https://devguide.python.org/developer-workflow/sbom' for more information.") + sys.exit(1) + + def filter_gitignored_paths(paths: list[str]) -> list[str]: """ Filter out paths excluded by the gitignore file. @@ -206,22 +214,47 @@ def main() -> None: discover_pip_sbom_package(sbom_data) # Ensure all packages in this tool are represented also in the SBOM file. - assert {package["name"] for package in sbom_data["packages"]} == set(PACKAGE_TO_FILES) + error_if( + {package["name"] for package in sbom_data["packages"]} != set(PACKAGE_TO_FILES), + "Packages defined in SBOM tool don't match those defined in SBOM file.", + ) # Make a bunch of assertions about the SBOM data to ensure it's consistent. for package in sbom_data["packages"]: - # Properties and ID must be properly formed. - assert set(package.keys()) == REQUIRED_PROPERTIES_PACKAGE - assert package["SPDXID"] == spdx_id(f"SPDXRef-PACKAGE-{package['name']}") + error_if( + "name" not in package, + "Package is missing the 'name' field" + ) + error_if( + set(package.keys()) != REQUIRED_PROPERTIES_PACKAGE, + f"Package '{package['name']}' is missing required fields", + ) + error_if( + package["SPDXID"] != spdx_id(f"SPDXRef-PACKAGE-{package['name']}"), + f"Package '{package['name']}' has a malformed SPDXID", + ) # Version must be in the download and external references. version = package["versionInfo"] - assert version in package["downloadLocation"] - assert all(version in ref["referenceLocator"] for ref in package["externalRefs"]) + error_if( + version not in package["downloadLocation"], + f"Version '{version}' for package '{package['name']} not in 'downloadLocation' field", + ) + error_if( + any(version not in ref["referenceLocator"] for ref in package["externalRefs"]), + ( + f"Version '{version}' for package '{package['name']} not in " + f"all 'externalRefs[].referenceLocator' fields" + ), + ) # License must be on the approved list for SPDX. - assert package["licenseConcluded"] in ALLOWED_LICENSE_EXPRESSIONS, package["licenseConcluded"] + license_concluded = package["licenseConcluded"] + error_if( + license_concluded not in ALLOWED_LICENSE_EXPRESSIONS, + f"License identifier '{license_concluded}' not in SBOM tool allowlist" + ) # Regenerate file information from current data. sbom_files = [] @@ -232,11 +265,13 @@ def main() -> None: package_spdx_id = spdx_id(f"SPDXRef-PACKAGE-{name}") exclude = files.exclude or () for include in sorted(files.include): - # Find all the paths and then filter them through .gitignore. paths = glob.glob(include, root_dir=CPYTHON_ROOT_DIR, recursive=True) paths = filter_gitignored_paths(paths) - assert paths, include # Make sure that every value returns something! + error_if( + len(paths) == 0, + f"No valid paths found at path '{include}' for package '{name}", + ) for path in paths: # Skip directories and excluded files