move project sources to src/ subdir (#12)

* move the distutils shim module back to a plain module * MANIFEST.in, the source layout, and pytest's default sys.path behavior conspired to mask a busted package, even though we were testing against the built package. `pytest` adds cwd to the path first, and since the `cffi` is in the root (grr), even though we're testing against the installed wheel, pytest loaded the subpackage from the source, masking that it was missing from the actual package (due to flat inclusions in MANIFEST.in). This is a longer-term problem that should be addressed by moving to a standard `src/` layout, and possibly also by bringing `_cffi_backend` in as a subpackage, but there are likely many dragons there with wheels that assume the presence of the top-level package. * move project sources under src/
python-cffi · Sep 26, 2023 · 932eba2 · 932eba2
1 parent 475c466
commit 932eba2
Show file tree

Hide file tree

Showing 65 changed files with 35 additions and 32 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -54,51 +54,51 @@ jobs:
 
         - spec: cp38-manylinux_aarch64
           foreign_arch: true
-          test_args: '{project}/c'
+          test_args: '{project}/src/c'
         - spec: cp39-manylinux_aarch64
           foreign_arch: true
-          test_args: '{project}/c'
+          test_args: '{project}/src/c'
         - spec: cp310-manylinux_aarch64
           foreign_arch: true
-          test_args: '{project}/c'
+          test_args: '{project}/src/c'
         - spec: cp311-manylinux_aarch64
           foreign_arch: true
-          test_args: '{project}/c'
+          test_args: '{project}/src/c'
         - spec: cp312-manylinux_aarch64
           foreign_arch: true
-          test_args: '{project}/c'
+          test_args: '{project}/src/c'
 
         - spec: cp38-manylinux_ppc64le
           foreign_arch: true
-          test_args: '{project}/c'
+          test_args: '{project}/src/c'
         - spec: cp39-manylinux_ppc64le
           foreign_arch: true
-          test_args: '{project}/c'
+          test_args: '{project}/src/c'
         - spec: cp310-manylinux_ppc64le
           foreign_arch: true
-          test_args: '{project}/c'
+          test_args: '{project}/src/c'
         - spec: cp311-manylinux_ppc64le
           foreign_arch: true
-          test_args: '{project}/c'
+          test_args: '{project}/src/c'
         - spec: cp312-manylinux_ppc64le
           foreign_arch: true
-          test_args: '{project}/c'
+          test_args: '{project}/src/c'
 
         - spec: cp38-manylinux_s390x
           foreign_arch: true
-          test_args: '{project}/c'
+          test_args: '{project}/src/c'
         - spec: cp39-manylinux_s390x
           foreign_arch: true
-          test_args: '{project}/c'
+          test_args: '{project}/src/c'
         - spec: cp310-manylinux_s390x
           foreign_arch: true
-          test_args: '{project}/c'
+          test_args: '{project}/src/c'
         - spec: cp311-manylinux_s390x
           foreign_arch: true
-          test_args: '{project}/c'
+          test_args: '{project}/src/c'
         - spec: cp312-manylinux_s390x
           foreign_arch: true
-          test_args: '{project}/c'
+          test_args: '{project}/src/c'
 
     steps:
     - name: clone repo
@@ -261,9 +261,9 @@ jobs:
         CIBW_BUILD: ${{ matrix.spec }}
         CIBW_PRERELEASE_PYTHONS: 'True'
         CIBW_TEST_REQUIRES: pytest setuptools
-        CIBW_TEST_COMMAND: 'python -m pytest {project}/c'
+        CIBW_TEST_COMMAND: 'python -m pytest {project}/src/c'
         # FIXME: /testing takes ~45min on Windows and has some failures...
-        # CIBW_TEST_COMMAND='python -m pytest {project}/c {project}/testing'
+        # CIBW_TEST_COMMAND='python -m pytest {project}/src/c {project}/testing'
       run: |
         python -m pip install --upgrade pip
         pip install "${{ matrix.cibw_version || 'cibuildwheel'}}"

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -1,5 +1,5 @@
-recursive-include cffi *.py *.h
-recursive-include c *.c *.h *.asm *.py win64.obj ffi.lib
+recursive-include src/cffi *.py *.h
+recursive-include src/c *.c *.h *.asm *.py win64.obj ffi.lib
 recursive-include testing *.py *.c *.h
 recursive-include doc *.py *.rst Makefile *.bat
 recursive-include demo py.cleanup *.py embedding_test.c manual.c

diff --git a/setup.py b/setup.py
@@ -7,7 +7,7 @@
 import setuptools
 
 
-sources = ['c/_cffi_backend.c']
+sources = ['src/c/_cffi_backend.c']
 libraries = ['ffi']
 include_dirs = ['/usr/include/ffi',
                 '/usr/include/libffi']    # may be changed by pkg-config
@@ -125,10 +125,10 @@ def use_homebrew_for_libffi():
 
 if sys.platform == "win32" and uses_msvc():
     if platform.machine() == "ARM64":
-        include_dirs.append(os.path.join("c/libffi_arm64/include"))
-        library_dirs.append(os.path.join("c/libffi_arm64"))
+        include_dirs.append(os.path.join("src/c/libffi_arm64/include"))
+        library_dirs.append(os.path.join("src/c/libffi_arm64"))
     else:
-        COMPILE_LIBFFI = 'c/libffi_x86_x64'    # from the CPython distribution
+        COMPILE_LIBFFI = 'src/c/libffi_x86_x64'    # from the CPython distribution
         assert os.path.isdir(COMPILE_LIBFFI), "directory not found!"
         include_dirs[:] = [COMPILE_LIBFFI]
         libraries[:] = []
@@ -198,6 +198,7 @@ def has_ext_modules(self):
         version='1.16.0rc1',
         python_requires='>=3.8',
         packages=['cffi'] if cpython else [],
+        package_dir={"": "src"},
         package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h', 
                                '_embedding.h', '_cffi_errors.h']}
                      if cpython else {},

diff --git a/c/_cffi_backend.c → src/c/_cffi_backend.c b/c/_cffi_backend.c → src/c/_cffi_backend.c
diff --git a/c/call_python.c → src/c/call_python.c b/c/call_python.c → src/c/call_python.c
diff --git a/c/cdlopen.c → src/c/cdlopen.c b/c/cdlopen.c → src/c/cdlopen.c
diff --git a/c/cffi1_module.c → src/c/cffi1_module.c b/c/cffi1_module.c → src/c/cffi1_module.c
diff --git a/c/cglob.c → src/c/cglob.c b/c/cglob.c → src/c/cglob.c
diff --git a/c/commontypes.c → src/c/commontypes.c b/c/commontypes.c → src/c/commontypes.c
diff --git a/c/ffi_obj.c → src/c/ffi_obj.c b/c/ffi_obj.c → src/c/ffi_obj.c
diff --git a/c/file_emulator.h → src/c/file_emulator.h b/c/file_emulator.h → src/c/file_emulator.h
diff --git a/c/lib_obj.c → src/c/lib_obj.c b/c/lib_obj.c → src/c/lib_obj.c
diff --git a/c/libffi_arm64/README → src/c/libffi_arm64/README b/c/libffi_arm64/README → src/c/libffi_arm64/README
diff --git a/c/libffi_arm64/build_libffi.bat → src/c/libffi_arm64/build_libffi.bat b/c/libffi_arm64/build_libffi.bat → src/c/libffi_arm64/build_libffi.bat
diff --git a/c/libffi_arm64/ffi.lib → src/c/libffi_arm64/ffi.lib b/c/libffi_arm64/ffi.lib → src/c/libffi_arm64/ffi.lib
diff --git a/c/libffi_arm64/include/ffi.h → src/c/libffi_arm64/include/ffi.h b/c/libffi_arm64/include/ffi.h → src/c/libffi_arm64/include/ffi.h
diff --git a/c/libffi_arm64/include/fficonfig.h → src/c/libffi_arm64/include/fficonfig.h b/c/libffi_arm64/include/fficonfig.h → src/c/libffi_arm64/include/fficonfig.h
diff --git a/c/libffi_arm64/include/ffitarget.h → src/c/libffi_arm64/include/ffitarget.h b/c/libffi_arm64/include/ffitarget.h → src/c/libffi_arm64/include/ffitarget.h
diff --git a/c/libffi_x86_x64/LICENSE → src/c/libffi_x86_x64/LICENSE b/c/libffi_x86_x64/LICENSE → src/c/libffi_x86_x64/LICENSE
diff --git a/c/libffi_x86_x64/README → src/c/libffi_x86_x64/README b/c/libffi_x86_x64/README → src/c/libffi_x86_x64/README
diff --git a/c/libffi_x86_x64/README.ctypes → src/c/libffi_x86_x64/README.ctypes b/c/libffi_x86_x64/README.ctypes → src/c/libffi_x86_x64/README.ctypes
diff --git a/c/libffi_x86_x64/ffi.c → src/c/libffi_x86_x64/ffi.c b/c/libffi_x86_x64/ffi.c → src/c/libffi_x86_x64/ffi.c
diff --git a/c/libffi_x86_x64/ffi.h → src/c/libffi_x86_x64/ffi.h b/c/libffi_x86_x64/ffi.h → src/c/libffi_x86_x64/ffi.h
diff --git a/c/libffi_x86_x64/ffi_common.h → src/c/libffi_x86_x64/ffi_common.h b/c/libffi_x86_x64/ffi_common.h → src/c/libffi_x86_x64/ffi_common.h
diff --git a/c/libffi_x86_x64/fficonfig.h → src/c/libffi_x86_x64/fficonfig.h b/c/libffi_x86_x64/fficonfig.h → src/c/libffi_x86_x64/fficonfig.h
diff --git a/c/libffi_x86_x64/ffitarget.h → src/c/libffi_x86_x64/ffitarget.h b/c/libffi_x86_x64/ffitarget.h → src/c/libffi_x86_x64/ffitarget.h
diff --git a/c/libffi_x86_x64/prep_cif.c → src/c/libffi_x86_x64/prep_cif.c b/c/libffi_x86_x64/prep_cif.c → src/c/libffi_x86_x64/prep_cif.c
diff --git a/c/libffi_x86_x64/types.c → src/c/libffi_x86_x64/types.c b/c/libffi_x86_x64/types.c → src/c/libffi_x86_x64/types.c
diff --git a/c/libffi_x86_x64/win32.c → src/c/libffi_x86_x64/win32.c b/c/libffi_x86_x64/win32.c → src/c/libffi_x86_x64/win32.c
diff --git a/c/libffi_x86_x64/win64.asm → src/c/libffi_x86_x64/win64.asm b/c/libffi_x86_x64/win64.asm → src/c/libffi_x86_x64/win64.asm
diff --git a/c/libffi_x86_x64/win64.obj → src/c/libffi_x86_x64/win64.obj b/c/libffi_x86_x64/win64.obj → src/c/libffi_x86_x64/win64.obj
diff --git a/c/malloc_closure.h → src/c/malloc_closure.h b/c/malloc_closure.h → src/c/malloc_closure.h
diff --git a/c/minibuffer.h → src/c/minibuffer.h b/c/minibuffer.h → src/c/minibuffer.h
diff --git a/c/misc_thread_common.h → src/c/misc_thread_common.h b/c/misc_thread_common.h → src/c/misc_thread_common.h
diff --git a/c/misc_thread_posix.h → src/c/misc_thread_posix.h b/c/misc_thread_posix.h → src/c/misc_thread_posix.h
diff --git a/c/misc_win32.h → src/c/misc_win32.h b/c/misc_win32.h → src/c/misc_win32.h
diff --git a/c/parse_c_type.c → src/c/parse_c_type.c b/c/parse_c_type.c → src/c/parse_c_type.c
diff --git a/c/realize_c_type.c → src/c/realize_c_type.c b/c/realize_c_type.c → src/c/realize_c_type.c
diff --git a/c/test_c.py → src/c/test_c.py b/c/test_c.py → src/c/test_c.py
diff --git a/c/wchar_helper.h → src/c/wchar_helper.h b/c/wchar_helper.h → src/c/wchar_helper.h
diff --git a/c/wchar_helper_3.h → src/c/wchar_helper_3.h b/c/wchar_helper_3.h → src/c/wchar_helper_3.h
diff --git a/cffi/__init__.py → src/cffi/__init__.py b/cffi/__init__.py → src/cffi/__init__.py
diff --git a/cffi/_cffi_errors.h → src/cffi/_cffi_errors.h b/cffi/_cffi_errors.h → src/cffi/_cffi_errors.h
diff --git a/cffi/_cffi_include.h → src/cffi/_cffi_include.h b/cffi/_cffi_include.h → src/cffi/_cffi_include.h
diff --git a/cffi/_embedding.h → src/cffi/_embedding.h b/cffi/_embedding.h → src/cffi/_embedding.h
diff --git a/cffi/_imp_emulation.py → src/cffi/_imp_emulation.py b/cffi/_imp_emulation.py → src/cffi/_imp_emulation.py
diff --git a/cffi/_shimmed_dist_utils/__init__.py → src/cffi/_shimmed_dist_utils.py b/cffi/_shimmed_dist_utils/__init__.py → src/cffi/_shimmed_dist_utils.py
diff --git a/cffi/api.py → src/cffi/api.py b/cffi/api.py → src/cffi/api.py
diff --git a/cffi/backend_ctypes.py → src/cffi/backend_ctypes.py b/cffi/backend_ctypes.py → src/cffi/backend_ctypes.py
diff --git a/cffi/cffi_opcode.py → src/cffi/cffi_opcode.py b/cffi/cffi_opcode.py → src/cffi/cffi_opcode.py
diff --git a/cffi/commontypes.py → src/cffi/commontypes.py b/cffi/commontypes.py → src/cffi/commontypes.py
diff --git a/cffi/cparser.py → src/cffi/cparser.py b/cffi/cparser.py → src/cffi/cparser.py
diff --git a/cffi/error.py → src/cffi/error.py b/cffi/error.py → src/cffi/error.py
diff --git a/cffi/ffiplatform.py → src/cffi/ffiplatform.py b/cffi/ffiplatform.py → src/cffi/ffiplatform.py
diff --git a/cffi/lock.py → src/cffi/lock.py b/cffi/lock.py → src/cffi/lock.py
diff --git a/cffi/model.py → src/cffi/model.py b/cffi/model.py → src/cffi/model.py
diff --git a/cffi/parse_c_type.h → src/cffi/parse_c_type.h b/cffi/parse_c_type.h → src/cffi/parse_c_type.h
diff --git a/cffi/pkgconfig.py → src/cffi/pkgconfig.py b/cffi/pkgconfig.py → src/cffi/pkgconfig.py
diff --git a/cffi/recompiler.py → src/cffi/recompiler.py b/cffi/recompiler.py → src/cffi/recompiler.py
diff --git a/cffi/setuptools_ext.py → src/cffi/setuptools_ext.py b/cffi/setuptools_ext.py → src/cffi/setuptools_ext.py
diff --git a/cffi/vengine_cpy.py → src/cffi/vengine_cpy.py b/cffi/vengine_cpy.py → src/cffi/vengine_cpy.py
diff --git a/cffi/vengine_gen.py → src/cffi/vengine_gen.py b/cffi/vengine_gen.py → src/cffi/vengine_gen.py
diff --git a/cffi/verifier.py → src/cffi/verifier.py b/cffi/verifier.py → src/cffi/verifier.py
diff --git a/testing/cffi0/test_version.py b/testing/cffi0/test_version.py
@@ -1,6 +1,7 @@
 import os, sys
 import pytest
 import cffi, _cffi_backend
+from pathlib import Path
 
 def setup_module(mod):
     if '_cffi_backend' in sys.builtin_module_names:
@@ -25,33 +26,33 @@ def test_version():
     assert v == _cffi_backend.__version__
 
 def test_doc_version():
-    parent = os.path.dirname(os.path.dirname(cffi.__file__))
-    p = os.path.join(parent, 'doc', 'source', 'conf.py')
+    cffi_root = Path(os.path.dirname(__file__)).parent.parent
+    p = cffi_root / 'doc/source/conf.py'
     content = open(p).read()
     #
     v = cffi.__version__
     assert ("version = '%s'\n" % v[:4]) in content
     assert ("release = '%s'\n" % v) in content
 
 def test_setup_version():
-    parent = os.path.dirname(os.path.dirname(cffi.__file__))
-    p = os.path.join(parent, 'setup.py')
+    cffi_root = Path(os.path.dirname(__file__)).parent.parent
+    p = cffi_root / 'setup.py'
     content = open(p).read()
     #
     v = cffi.__version__.replace('+', '')
     assert ("version='%s'" % v) in content
 
 def test_c_version():
-    parent = os.path.dirname(os.path.dirname(cffi.__file__))
+    cffi_root = Path(os.path.dirname(__file__)).parent.parent
     v = cffi.__version__
-    p = os.path.join(parent, 'c', 'test_c.py')
+    p = cffi_root / 'src/c/test_c.py'
     content = open(p).read()
     #v = BACKEND_VERSIONS.get(v, v)
     assert (('assert __version__ == "%s"' % v) in content)
 
 def test_embedding_h():
-    parent = os.path.dirname(os.path.dirname(cffi.__file__))
+    cffi_root = Path(os.path.dirname(__file__)).parent.parent
     v = cffi.__version__
-    p = os.path.join(parent, 'cffi', '_embedding.h')
+    p = cffi_root / 'src/cffi/_embedding.h'
     content = open(p).read()
     assert ('cffi version: %s"' % (v,)) in content
diff --git a/testing/cffi1/test_parse_c_type.py b/testing/cffi1/test_parse_c_type.py
@@ -2,6 +2,7 @@
 import pytest
 import cffi
 from cffi import cffi_opcode
+from pathlib import Path
 
 if '__pypy__' in sys.builtin_module_names:
     try:
@@ -11,7 +12,7 @@
         # older pytest
         pytest.skip("not available on pypy")
 
-cffi_dir = os.path.dirname(cffi_opcode.__file__)
+cffi_dir = str(Path(os.path.dirname(__file__)).parent.parent / "src/cffi")
 
 r_macro = re.compile(r"#define \w+[(][^\n]*|#include [^\n]*")
 r_define = re.compile(r"(#define \w+) [^\n]*")