diff --git a/.gitignore b/.gitignore index 9f79dea..de16b11 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,8 @@ __pycache__/ *dist* *.egg-info +**/.DS_Store + # Files related to code coverage stats **/.coverage diff --git a/pybind11/.clang-tidy b/pybind11/.clang-tidy index d945a4a..2301838 100644 --- a/pybind11/.clang-tidy +++ b/pybind11/.clang-tidy @@ -63,6 +63,8 @@ Checks: | -bugprone-unused-raii, CheckOptions: +- key: modernize-use-equals-default.IgnoreMacros + value: false - key: performance-for-range-copy.WarnOnAllAutoCopies value: true - key: performance-inefficient-string-concatenation.StrictMode diff --git a/pybind11/.codespell-ignore-lines b/pybind11/.codespell-ignore-lines new file mode 100644 index 0000000..2a01d63 --- /dev/null +++ b/pybind11/.codespell-ignore-lines @@ -0,0 +1,24 @@ +template + template + auto &this_ = static_cast(*this); + if (load_impl(temp, false)) { + ssize_t nd = 0; + auto trivial = broadcast(buffers, nd, shape); + auto ndim = (size_t) nd; + int nd; + ssize_t ndim() const { return detail::array_proxy(m_ptr)->nd; } + using op = op_impl; +template + template + class_ &def(const detail::op_ &op, const Extra &...extra) { + class_ &def_cast(const detail::op_ &op, const Extra &...extra) { +@pytest.mark.parametrize("access", ["ro", "rw", "static_ro", "static_rw"]) +struct IntStruct { + explicit IntStruct(int v) : value(v){}; + ~IntStruct() { value = -value; } + IntStruct(const IntStruct &) = default; + IntStruct &operator=(const IntStruct &) = default; + py::class_(m, "IntStruct").def(py::init([](const int i) { return IntStruct(i); })); + py::implicitly_convertible(); + m.def("test", [](int expected, const IntStruct &in) { + [](int expected, const IntStruct &in) { diff --git a/pybind11/.github/CONTRIBUTING.md b/pybind11/.github/CONTRIBUTING.md index 00b1fea..ad79743 100644 --- a/pybind11/.github/CONTRIBUTING.md +++ b/pybind11/.github/CONTRIBUTING.md @@ -235,8 +235,8 @@ directory inside your pybind11 git clone. Files will be modified in place, so you can use git to monitor the changes. ```bash -docker run --rm -v $PWD:/mounted_pybind11 -it silkeh/clang:13 -apt-get update && apt-get install -y python3-dev python3-pytest +docker run --rm -v $PWD:/mounted_pybind11 -it silkeh/clang:15-bullseye +apt-get update && apt-get install -y git python3-dev python3-pytest cmake -S /mounted_pybind11/ -B build -DCMAKE_CXX_CLANG_TIDY="$(which clang-tidy);--use-color" -DDOWNLOAD_EIGEN=ON -DDOWNLOAD_CATCH=ON -DCMAKE_CXX_STANDARD=17 cmake --build build -j 2 ``` diff --git a/pybind11/.github/ISSUE_TEMPLATE/bug-report.yml b/pybind11/.github/ISSUE_TEMPLATE/bug-report.yml index bd6a9a8..4f1e78f 100644 --- a/pybind11/.github/ISSUE_TEMPLATE/bug-report.yml +++ b/pybind11/.github/ISSUE_TEMPLATE/bug-report.yml @@ -6,7 +6,8 @@ body: - type: markdown attributes: value: | - Maintainers will only make a best effort to triage PRs. Please do your best to make the issue as easy to act on as possible, and only open if clearly a problem with pybind11 (ask first if unsure). + Please do your best to make the issue as easy to act on as possible, and only submit here if there is clearly a problem with pybind11 (ask first if unsure). **Note that a reproducer in a PR is much more likely to get immediate attention.** + - type: checkboxes id: steps attributes: @@ -20,6 +21,13 @@ body: - label: Consider asking first in the [Gitter chat room](https://gitter.im/pybind/Lobby) or in a [Discussion](https:/pybind/pybind11/discussions/new). required: false + - type: input + id: version + attributes: + label: What version (or hash if on master) of pybind11 are you using? + validations: + required: true + - type: textarea id: description attributes: @@ -40,6 +48,14 @@ body: The code should be minimal, have no external dependencies, isolate the function(s) that cause breakage. Submit matched and complete C++ and Python snippets that can be easily compiled and run to diagnose the - issue. If possible, make a PR with a new, failing test to give us a - starting point to work on! + issue. β€” Note that a reproducer in a PR is much more likely to get + immediate attention: failing tests in the pybind11 CI are the best + starting point for working out fixes. render: text + + - type: input + id: regression + attributes: + label: Is this a regression? Put the last known working version here if it is. + description: Put the last known working version here if this is a regression. + value: Not a regression diff --git a/pybind11/.github/workflows/ci.yml b/pybind11/.github/workflows/ci.yml index 412282a..48f7c5e 100644 --- a/pybind11/.github/workflows/ci.yml +++ b/pybind11/.github/workflows/ci.yml @@ -9,14 +9,19 @@ on: - stable - v* +permissions: read-all + concurrency: group: test-${{ github.ref }} cancel-in-progress: true env: + PIP_BREAK_SYSTEM_PACKAGES: 1 PIP_ONLY_BINARY: numpy FORCE_COLOR: 3 PYTEST_TIMEOUT: 300 + # For cmake: + VERBOSE: 1 jobs: # This is the "main" test suite, which tests a large number of different @@ -25,15 +30,16 @@ jobs: strategy: fail-fast: false matrix: - runs-on: [ubuntu-latest, windows-2022, macos-latest] + runs-on: [ubuntu-20.04, windows-2022, macos-latest] python: - '3.6' - '3.9' - '3.10' - - '3.11-dev' - - 'pypy-3.7' + - '3.11' + - '3.12' - 'pypy-3.8' - 'pypy-3.9' + - 'pypy-3.10' # Items in here will either be added to the build matrix (if not # present), or add new keys to an existing matrix element if all the @@ -42,12 +48,12 @@ jobs: # We support an optional key: args, for cmake args include: # Just add a key - - runs-on: ubuntu-latest + - runs-on: ubuntu-20.04 python: '3.6' args: > -DPYBIND11_FINDPYTHON=ON -DCMAKE_CXX_FLAGS="-D_=1" - - runs-on: ubuntu-latest + - runs-on: ubuntu-20.04 python: 'pypy-3.8' args: > -DPYBIND11_FINDPYTHON=ON @@ -69,6 +75,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} + allow-prereleases: true - name: Setup Boost (Linux) # Can't use boost + define _ @@ -80,7 +87,7 @@ jobs: run: brew install boost - name: Update CMake - uses: jwlawson/actions-setup-cmake@v1.12 + uses: jwlawson/actions-setup-cmake@v1.14 - name: Cache wheels if: runner.os == 'macOS' @@ -102,10 +109,12 @@ jobs: run: python -m pip install pytest-github-actions-annotate-failures # First build - C++11 mode and inplace + # More-or-less randomly adding -DPYBIND11_SIMPLE_GIL_MANAGEMENT=ON here. - name: Configure C++11 ${{ matrix.args }} run: > cmake -S . -B . -DPYBIND11_WERROR=ON + -DPYBIND11_SIMPLE_GIL_MANAGEMENT=ON -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON -DCMAKE_CXX_STANDARD=11 @@ -119,7 +128,7 @@ jobs: - name: C++11 tests # TODO: Figure out how to load the DLL on Python 3.8+ - if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10' || matrix.python == '3.11-dev' || matrix.python == 'pypy-3.8'))" + if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10' || matrix.python == '3.11' || matrix.python == 'pypy-3.8'))" run: cmake --build . --target cpptest -j 2 - name: Interface test C++11 @@ -129,10 +138,12 @@ jobs: run: git clean -fdx # Second build - C++17 mode and in a build directory + # More-or-less randomly adding -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF here. - name: Configure C++17 run: > cmake -S . -B build2 -DPYBIND11_WERROR=ON + -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON -DCMAKE_CXX_STANDARD=17 @@ -146,7 +157,7 @@ jobs: - name: C++ tests # TODO: Figure out how to load the DLL on Python 3.8+ - if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10' || matrix.python == '3.11-dev' || matrix.python == 'pypy-3.8'))" + if: "!(runner.os == 'Windows' && (matrix.python == 3.8 || matrix.python == 3.9 || matrix.python == '3.10' || matrix.python == '3.11' || matrix.python == 'pypy-3.8'))" run: cmake --build build2 --target cpptest # Third build - C++17 mode with unstable ABI @@ -158,7 +169,6 @@ jobs: -DDOWNLOAD_EIGEN=ON -DCMAKE_CXX_STANDARD=17 -DPYBIND11_INTERNALS_VERSION=10000000 - "-DPYBIND11_TEST_OVERRIDE=test_call_policies.cpp;test_gil_scoped.cpp;test_thread.cpp" ${{ matrix.args }} - name: Build (unstable ABI) @@ -173,7 +183,9 @@ jobs: # This makes sure the setup_helpers module can build packages using # setuptools - name: Setuptools helpers test - run: pytest tests/extra_setuptools + run: | + pip install setuptools + pytest tests/extra_setuptools if: "!(matrix.runs-on == 'windows-2022')" @@ -186,23 +198,23 @@ jobs: - python-version: "3.9" python-debug: true valgrind: true - - python-version: "3.11-dev" + - python-version: "3.11" python-debug: false name: "🐍 ${{ matrix.python-version }}${{ matrix.python-debug && '-dbg' || '' }} (deadsnakes)${{ matrix.valgrind && ' β€’ Valgrind' || '' }} β€’ x64" - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v3 - name: Setup Python ${{ matrix.python-version }} (deadsnakes) - uses: deadsnakes/action@v2.1.1 + uses: deadsnakes/action@v3.0.1 with: python-version: ${{ matrix.python-version }} debug: ${{ matrix.python-debug }} - name: Update CMake - uses: jwlawson/actions-setup-cmake@v1.12 + uses: jwlawson/actions-setup-cmake@v1.14 - name: Valgrind cache if: matrix.valgrind @@ -235,8 +247,6 @@ jobs: python -m pip install -r tests/requirements.txt - name: Configure - env: - SETUPTOOLS_USE_DISTUTILS: stdlib run: > cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug @@ -274,18 +284,27 @@ jobs: - dev std: - 11 + container_suffix: + - "" include: - clang: 5 std: 14 - - clang: 10 - std: 20 - clang: 10 std: 17 + - clang: 11 + std: 20 + - clang: 12 + std: 20 + - clang: 13 + std: 20 - clang: 14 std: 20 + - clang: 15 + std: 20 + container_suffix: "-bullseye" name: "🐍 3 β€’ Clang ${{ matrix.clang }} β€’ C++${{ matrix.std }} β€’ x64" - container: "silkeh/clang:${{ matrix.clang }}" + container: "silkeh/clang:${{ matrix.clang }}${{ matrix.container_suffix }}" steps: - uses: actions/checkout@v3 @@ -318,8 +337,8 @@ jobs: # Testing NVCC; forces sources to behave like .cu files cuda: runs-on: ubuntu-latest - name: "🐍 3.8 β€’ CUDA 11.2 β€’ Ubuntu 20.04" - container: nvidia/cuda:11.2.2-devel-ubuntu20.04 + name: "🐍 3.10 β€’ CUDA 12.2 β€’ Ubuntu 22.04" + container: nvidia/cuda:12.2.0-devel-ubuntu22.04 steps: - uses: actions/checkout@v3 @@ -384,8 +403,9 @@ jobs: # Testing on CentOS 7 + PGI compilers, which seems to require more workarounds centos-nvhpc7: + if: ${{ false }} # JOB DISABLED (NEEDS WORK): https://github.com/pybind/pybind11/issues/4690 runs-on: ubuntu-latest - name: "🐍 3 β€’ CentOS7 / PGI 22.3 β€’ x64" + name: "🐍 3 β€’ CentOS7 / PGI 22.9 β€’ x64" container: centos:7 steps: @@ -395,7 +415,7 @@ jobs: run: yum update -y && yum install -y epel-release && yum install -y git python3-devel make environment-modules cmake3 yum-utils - name: Install NVidia HPC SDK - run: yum-config-manager --add-repo https://developer.download.nvidia.com/hpc-sdk/rhel/nvhpc.repo && yum -y install nvhpc-22.3 + run: yum-config-manager --add-repo https://developer.download.nvidia.com/hpc-sdk/rhel/nvhpc.repo && yum -y install nvhpc-22.9 # On CentOS 7, we have to filter a few tests (compiler internal error) # and allow deeper template recursion (not needed on CentOS 8 with a newer @@ -405,12 +425,12 @@ jobs: shell: bash run: | source /etc/profile.d/modules.sh - module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/22.3 + module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/22.9 cmake3 -S . -B build -DDOWNLOAD_CATCH=ON \ -DCMAKE_CXX_STANDARD=11 \ -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") \ -DCMAKE_CXX_FLAGS="-Wc,--pending_instantiations=0" \ - -DPYBIND11_TEST_FILTER="test_smart_ptr.cpp;test_virtual_functions.cpp" + -DPYBIND11_TEST_FILTER="test_smart_ptr.cpp" # Building before installing Pip should produce a warning but not an error - name: Build @@ -437,14 +457,14 @@ jobs: strategy: fail-fast: false matrix: - gcc: - - 7 - - latest - std: - - 11 include: - - gcc: 10 - std: 20 + - { gcc: 7, std: 11 } + - { gcc: 7, std: 17 } + - { gcc: 8, std: 14 } + - { gcc: 8, std: 17 } + - { gcc: 10, std: 17 } + - { gcc: 11, std: 20 } + - { gcc: 12, std: 20 } name: "🐍 3 β€’ GCC ${{ matrix.gcc }} β€’ C++${{ matrix.std }}β€’ x64" container: "gcc:${{ matrix.gcc }}" @@ -459,7 +479,7 @@ jobs: run: python3 -m pip install --upgrade pip - name: Update CMake - uses: jwlawson/actions-setup-cmake@v1.12 + uses: jwlawson/actions-setup-cmake@v1.14 - name: Configure shell: bash @@ -482,6 +502,24 @@ jobs: - name: Interface test run: cmake --build build --target test_cmake_build + - name: Configure - Exercise cmake -DPYBIND11_TEST_OVERRIDE + if: matrix.gcc == '12' + shell: bash + run: > + cmake -S . -B build_partial + -DPYBIND11_WERROR=ON + -DDOWNLOAD_CATCH=ON + -DCMAKE_CXX_STANDARD=${{ matrix.std }} + -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") + "-DPYBIND11_TEST_OVERRIDE=test_call_policies.cpp;test_gil_scoped.cpp;test_thread.cpp" + + - name: Build - Exercise cmake -DPYBIND11_TEST_OVERRIDE + if: matrix.gcc == '12' + run: cmake --build build_partial -j 2 + + - name: Python tests - Exercise cmake -DPYBIND11_TEST_OVERRIDE + if: matrix.gcc == '12' + run: cmake --build build_partial --target pytest # Testing on ICC using the oneAPI apt repo icc: @@ -731,6 +769,9 @@ jobs: args: -DCMAKE_CXX_STANDARD=20 - python: 3.8 args: -DCMAKE_CXX_STANDARD=17 + - python: 3.7 + args: -DCMAKE_CXX_STANDARD=14 + name: "🐍 ${{ matrix.python }} β€’ MSVC 2019 β€’ x86 ${{ matrix.args }}" runs-on: windows-2019 @@ -745,10 +786,10 @@ jobs: architecture: x86 - name: Update CMake - uses: jwlawson/actions-setup-cmake@v1.12 + uses: jwlawson/actions-setup-cmake@v1.14 - name: Prepare MSVC - uses: ilammy/msvc-dev-cmd@v1.10.0 + uses: ilammy/msvc-dev-cmd@v1.12.1 with: arch: x86 @@ -798,10 +839,10 @@ jobs: architecture: x86 - name: Update CMake - uses: jwlawson/actions-setup-cmake@v1.12 + uses: jwlawson/actions-setup-cmake@v1.14 - name: Prepare MSVC - uses: ilammy/msvc-dev-cmd@v1.10.0 + uses: ilammy/msvc-dev-cmd@v1.12.1 with: arch: x86 @@ -849,7 +890,7 @@ jobs: python3 -m pip install -r tests/requirements.txt - name: Update CMake - uses: jwlawson/actions-setup-cmake@v1.12 + uses: jwlawson/actions-setup-cmake@v1.14 - name: Configure C++20 run: > @@ -871,6 +912,21 @@ jobs: - name: Interface test C++20 run: cmake --build build --target test_cmake_build + - name: Configure C++20 - Exercise cmake -DPYBIND11_TEST_OVERRIDE + run: > + cmake -S . -B build_partial + -DPYBIND11_WERROR=ON + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DCMAKE_CXX_STANDARD=20 + "-DPYBIND11_TEST_OVERRIDE=test_call_policies.cpp;test_gil_scoped.cpp;test_thread.cpp" + + - name: Build C++20 - Exercise cmake -DPYBIND11_TEST_OVERRIDE + run: cmake --build build_partial -j 2 + + - name: Python tests - Exercise cmake -DPYBIND11_TEST_OVERRIDE + run: cmake --build build_partial --target pytest + mingw: name: "🐍 3 β€’ windows-latest β€’ ${{ matrix.sys }}" runs-on: windows-latest @@ -905,7 +961,7 @@ jobs: - name: Configure C++11 # LTO leads to many undefined reference like # `pybind11::detail::function_call::function_call(pybind11::detail::function_call&&) - run: cmake -G "MinGW Makefiles" -DCMAKE_CXX_STANDARD=11 -DDOWNLOAD_CATCH=ON -S . -B build + run: cmake -G "MinGW Makefiles" -DCMAKE_CXX_STANDARD=11 -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -S . -B build - name: Build C++11 run: cmake --build build -j 2 @@ -923,7 +979,7 @@ jobs: run: git clean -fdx - name: Configure C++14 - run: cmake -G "MinGW Makefiles" -DCMAKE_CXX_STANDARD=14 -DDOWNLOAD_CATCH=ON -S . -B build2 + run: cmake -G "MinGW Makefiles" -DCMAKE_CXX_STANDARD=14 -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -S . -B build2 - name: Build C++14 run: cmake --build build2 -j 2 @@ -941,7 +997,7 @@ jobs: run: git clean -fdx - name: Configure C++17 - run: cmake -G "MinGW Makefiles" -DCMAKE_CXX_STANDARD=17 -DDOWNLOAD_CATCH=ON -S . -B build3 + run: cmake -G "MinGW Makefiles" -DCMAKE_CXX_STANDARD=17 -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -S . -B build3 - name: Build C++17 run: cmake --build build3 -j 2 @@ -954,3 +1010,156 @@ jobs: - name: Interface test C++17 run: PYTHONHOME=/${{matrix.sys}} PYTHONPATH=/${{matrix.sys}} cmake --build build3 --target test_cmake_build + + windows_clang: + + strategy: + matrix: + os: [windows-latest] + python: ['3.10'] + + runs-on: "${{ matrix.os }}" + + name: "🐍 ${{ matrix.python }} β€’ ${{ matrix.os }} β€’ clang-latest" + + steps: + - name: Show env + run: env + + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Clang + uses: egor-tensin/setup-clang@v1 + + - name: Setup Python ${{ matrix.python }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + + - name: Update CMake + uses: jwlawson/actions-setup-cmake@v1.14 + + - name: Install ninja-build tool + uses: seanmiddleditch/gha-setup-ninja@v3 + + - name: Run pip installs + run: | + python -m pip install --upgrade pip + python -m pip install -r tests/requirements.txt + + - name: Show Clang++ version + run: clang++ --version + + - name: Show CMake version + run: cmake --version + + # TODO: WERROR=ON + - name: Configure Clang + run: > + cmake -G Ninja -S . -B . + -DPYBIND11_WERROR=OFF + -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DCMAKE_CXX_COMPILER=clang++ + -DCMAKE_CXX_STANDARD=17 + + - name: Build + run: cmake --build . -j 2 + + - name: Python tests + run: cmake --build . --target pytest -j 2 + + - name: C++ tests + run: cmake --build . --target cpptest -j 2 + + - name: Interface test + run: cmake --build . --target test_cmake_build -j 2 + + - name: Clean directory + run: git clean -fdx + + macos_brew_install_llvm: + name: "macos-latest β€’ brew install llvm" + runs-on: macos-latest + + env: + # https://apple.stackexchange.com/questions/227026/how-to-install-recent-clang-with-homebrew + LDFLAGS: '-L/usr/local/opt/llvm/lib -Wl,-rpath,/usr/local/opt/llvm/lib' + + steps: + - name: Update PATH + run: echo "/usr/local/opt/llvm/bin" >> $GITHUB_PATH + + - name: Show env + run: env + + - name: Checkout + uses: actions/checkout@v3 + + - name: Show Clang++ version before brew install llvm + run: clang++ --version + + - name: brew install llvm + run: brew install llvm + + - name: Show Clang++ version after brew install llvm + run: clang++ --version + + - name: Update CMake + uses: jwlawson/actions-setup-cmake@v1.14 + + - name: Run pip installs + run: | + python3 -m pip install --upgrade pip + python3 -m pip install -r tests/requirements.txt + python3 -m pip install numpy + python3 -m pip install scipy + + - name: Show CMake version + run: cmake --version + + - name: CMake Configure + run: > + cmake -S . -B . + -DPYBIND11_WERROR=ON + -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DCMAKE_CXX_COMPILER=clang++ + -DCMAKE_CXX_STANDARD=17 + -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") + + - name: Build + run: cmake --build . -j 2 + + - name: Python tests + run: cmake --build . --target pytest -j 2 + + - name: C++ tests + run: cmake --build . --target cpptest -j 2 + + - name: Interface test + run: cmake --build . --target test_cmake_build -j 2 + + - name: CMake Configure - Exercise cmake -DPYBIND11_TEST_OVERRIDE + run: > + cmake -S . -B build_partial + -DPYBIND11_WERROR=ON + -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DCMAKE_CXX_COMPILER=clang++ + -DCMAKE_CXX_STANDARD=17 + -DPYTHON_EXECUTABLE=$(python3 -c "import sys; print(sys.executable)") + "-DPYBIND11_TEST_OVERRIDE=test_call_policies.cpp;test_gil_scoped.cpp;test_thread.cpp" + + - name: Build - Exercise cmake -DPYBIND11_TEST_OVERRIDE + run: cmake --build build_partial -j 2 + + - name: Python tests - Exercise cmake -DPYBIND11_TEST_OVERRIDE + run: cmake --build build_partial --target pytest -j 2 + + - name: Clean directory + run: git clean -fdx diff --git a/pybind11/.github/workflows/configure.yml b/pybind11/.github/workflows/configure.yml index edcad41..ec7cd61 100644 --- a/pybind11/.github/workflows/configure.yml +++ b/pybind11/.github/workflows/configure.yml @@ -9,6 +9,14 @@ on: - stable - v* +permissions: + contents: read + +env: + PIP_BREAK_SYSTEM_PACKAGES: 1 + # For cmake: + VERBOSE: 1 + jobs: # This tests various versions of CMake in various combinations, to make sure # the configure step passes. @@ -16,22 +24,26 @@ jobs: strategy: fail-fast: false matrix: - runs-on: [ubuntu-latest, macos-latest, windows-latest] + runs-on: [ubuntu-20.04, macos-latest, windows-latest] arch: [x64] - cmake: ["3.23"] + cmake: ["3.26"] include: - - runs-on: ubuntu-latest + - runs-on: ubuntu-20.04 + arch: x64 + cmake: "3.5" + + - runs-on: ubuntu-20.04 arch: x64 - cmake: 3.4 + cmake: "3.27" - runs-on: macos-latest arch: x64 - cmake: 3.7 + cmake: "3.7" - runs-on: windows-2019 arch: x64 # x86 compilers seem to be missing on 2019 image - cmake: 3.18 + cmake: "3.18" name: 🐍 3.7 β€’ CMake ${{ matrix.cmake }} β€’ ${{ matrix.runs-on }} runs-on: ${{ matrix.runs-on }} @@ -51,7 +63,7 @@ jobs: # An action for adding a specific version of CMake: # https://github.com/jwlawson/actions-setup-cmake - name: Setup CMake ${{ matrix.cmake }} - uses: jwlawson/actions-setup-cmake@v1.12 + uses: jwlawson/actions-setup-cmake@v1.14 with: cmake-version: ${{ matrix.cmake }} diff --git a/pybind11/.github/workflows/format.yml b/pybind11/.github/workflows/format.yml index 31d893c..b8242ee 100644 --- a/pybind11/.github/workflows/format.yml +++ b/pybind11/.github/workflows/format.yml @@ -12,8 +12,13 @@ on: - stable - "v*" +permissions: + contents: read + env: FORCE_COLOR: 3 + # For cmake: + VERBOSE: 1 jobs: pre-commit: @@ -36,12 +41,12 @@ jobs: # in .github/CONTRIBUTING.md and update as needed. name: Clang-Tidy runs-on: ubuntu-latest - container: silkeh/clang:13 + container: silkeh/clang:15-bullseye steps: - uses: actions/checkout@v3 - name: Install requirements - run: apt-get update && apt-get install -y python3-dev python3-pytest + run: apt-get update && apt-get install -y git python3-dev python3-pytest - name: Configure run: > diff --git a/pybind11/.github/workflows/labeler.yml b/pybind11/.github/workflows/labeler.yml index d2b5979..858a4a0 100644 --- a/pybind11/.github/workflows/labeler.yml +++ b/pybind11/.github/workflows/labeler.yml @@ -3,14 +3,23 @@ on: pull_request_target: types: [closed] +permissions: {} + jobs: label: name: Labeler runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write steps: - uses: actions/labeler@main - if: github.event.pull_request.merged == true + if: > + github.event.pull_request.merged == true && + !startsWith(github.event.pull_request.title, 'chore(deps):') && + !startsWith(github.event.pull_request.title, 'ci(fix):') && + !startsWith(github.event.pull_request.title, 'docs(changelog):') with: repo-token: ${{ secrets.GITHUB_TOKEN }} configuration-path: .github/labeler_merged.yml diff --git a/pybind11/.github/workflows/pip.yml b/pybind11/.github/workflows/pip.yml index 2c16735..d6687b4 100644 --- a/pybind11/.github/workflows/pip.yml +++ b/pybind11/.github/workflows/pip.yml @@ -12,7 +12,11 @@ on: types: - published +permissions: + contents: read + env: + PIP_BREAK_SYSTEM_PACKAGES: 1 PIP_ONLY_BINARY: numpy jobs: @@ -98,13 +102,13 @@ jobs: - uses: actions/download-artifact@v3 - name: Publish standard package - uses: pypa/gh-action-pypi-publish@v1.5.0 + uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.pypi_password }} - packages_dir: standard/ + packages-dir: standard/ - name: Publish global package - uses: pypa/gh-action-pypi-publish@v1.5.0 + uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.pypi_password_global }} - packages_dir: global/ + packages-dir: global/ diff --git a/pybind11/.github/workflows/upstream.yml b/pybind11/.github/workflows/upstream.yml index a40a6c7..dd8a1c9 100644 --- a/pybind11/.github/workflows/upstream.yml +++ b/pybind11/.github/workflows/upstream.yml @@ -1,112 +1,116 @@ - name: Upstream on: workflow_dispatch: pull_request: +permissions: + contents: read + concurrency: group: upstream-${{ github.ref }} cancel-in-progress: true env: - PIP_ONLY_BINARY: numpy + PIP_BREAK_SYSTEM_PACKAGES: 1 + PIP_ONLY_BINARY: ":all:" + # For cmake: + VERBOSE: 1 jobs: standard: - name: "🐍 3.11 latest internals β€’ ubuntu-latest β€’ x64" + name: "🐍 3.12 latest β€’ ubuntu-latest β€’ x64" runs-on: ubuntu-latest + # Only runs when the 'python dev' label is selected if: "contains(github.event.pull_request.labels.*.name, 'python dev')" steps: - uses: actions/checkout@v3 - - name: Setup Python 3.11 + - name: Setup Python 3.12 uses: actions/setup-python@v4 with: - python-version: "3.11-dev" + python-version: "3.12-dev" - - name: Setup Boost (Linux) - if: runner.os == 'Linux' + - name: Setup Boost run: sudo apt-get install libboost-dev - name: Update CMake - uses: jwlawson/actions-setup-cmake@v1.12 + uses: jwlawson/actions-setup-cmake@v1.14 - - name: Prepare env + - name: Run pip installs run: | + python -m pip install --upgrade pip python -m pip install -r tests/requirements.txt - - name: Setup annotations on Linux - if: runner.os == 'Linux' - run: python -m pip install pytest-github-actions-annotate-failures + - name: Show platform info + run: | + python -m platform + cmake --version + pip list # First build - C++11 mode and inplace - name: Configure C++11 run: > - cmake -S . -B . + cmake -S . -B build11 -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON -DCMAKE_CXX_STANDARD=11 + -DCMAKE_BUILD_TYPE=Debug - name: Build C++11 - run: cmake --build . -j 2 + run: cmake --build build11 -j 2 - name: Python tests C++11 - run: cmake --build . --target pytest -j 2 + run: cmake --build build11 --target pytest -j 2 - name: C++11 tests - run: cmake --build . --target cpptest -j 2 + run: cmake --build build11 --target cpptest -j 2 - name: Interface test C++11 - run: cmake --build . --target test_cmake_build - - - name: Clean directory - run: git clean -fdx + run: cmake --build build11 --target test_cmake_build # Second build - C++17 mode and in a build directory - name: Configure C++17 run: > - cmake -S . -B build2 + cmake -S . -B build17 -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON -DCMAKE_CXX_STANDARD=17 - ${{ matrix.args }} - ${{ matrix.args2 }} - - name: Build - run: cmake --build build2 -j 2 + - name: Build C++17 + run: cmake --build build17 -j 2 - - name: Python tests - run: cmake --build build2 --target pytest + - name: Python tests C++17 + run: cmake --build build17 --target pytest - - name: C++ tests - run: cmake --build build2 --target cpptest + - name: C++17 tests + run: cmake --build build17 --target cpptest # Third build - C++17 mode with unstable ABI - name: Configure (unstable ABI) run: > - cmake -S . -B build3 + cmake -S . -B build17max -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON -DCMAKE_CXX_STANDARD=17 -DPYBIND11_INTERNALS_VERSION=10000000 - "-DPYBIND11_TEST_OVERRIDE=test_call_policies.cpp;test_gil_scoped.cpp;test_thread.cpp" - ${{ matrix.args }} - name: Build (unstable ABI) - run: cmake --build build3 -j 2 + run: cmake --build build17max -j 2 - name: Python tests (unstable ABI) - run: cmake --build build3 --target pytest + run: cmake --build build17max --target pytest - - name: Interface test - run: cmake --build build2 --target test_cmake_build + - name: Interface test (unstable ABI) + run: cmake --build build17max --target test_cmake_build # This makes sure the setup_helpers module can build packages using # setuptools - name: Setuptools helpers test - run: pytest tests/extra_setuptools + run: | + pip install setuptools + pytest tests/extra_setuptools diff --git a/pybind11/.gitignore b/pybind11/.gitignore index 3cf4fbb..43d5094 100644 --- a/pybind11/.gitignore +++ b/pybind11/.gitignore @@ -43,3 +43,4 @@ pybind11Targets.cmake /pybind11/share/* /docs/_build/* .ipynb_checkpoints/ +tests/main.cpp diff --git a/pybind11/.pre-commit-config.yaml b/pybind11/.pre-commit-config.yaml index ba9955a..86ac965 100644 --- a/pybind11/.pre-commit-config.yaml +++ b/pybind11/.pre-commit-config.yaml @@ -12,10 +12,62 @@ # # See https://github.com/pre-commit/pre-commit + +ci: + autoupdate_commit_msg: "chore(deps): update pre-commit hooks" + autofix_commit_msg: "style: pre-commit fixes" + autoupdate_schedule: monthly + +# third-party content +exclude: ^tools/JoinPaths.cmake$ + repos: + +# Clang format the codebase automatically +- repo: https://github.com/pre-commit/mirrors-clang-format + rev: "v16.0.6" + hooks: + - id: clang-format + types_or: [c++, c, cuda] + +# Black, the code formatter, natively supports pre-commit +- repo: https://github.com/psf/black + rev: "23.3.0" # Keep in sync with blacken-docs + hooks: + - id: black + +# Ruff, the Python auto-correcting linter written in Rust +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.0.276 + hooks: + - id: ruff + args: ["--fix", "--show-fixes"] + +# Check static types with mypy +- repo: https://github.com/pre-commit/mirrors-mypy + rev: "v1.4.1" + hooks: + - id: mypy + args: [] + exclude: ^(tests|docs)/ + additional_dependencies: + - markdown-it-py<3 # Drop this together with dropping Python 3.7 support. + - nox + - rich + - types-setuptools + +# CMake formatting +- repo: https://github.com/cheshirekow/cmake-format-precommit + rev: "v0.6.13" + hooks: + - id: cmake-format + additional_dependencies: [pyyaml] + types: [file] + files: (\.cmake|CMakeLists.txt)(.in)?$ + # Standard hooks - repo: https://github.com/pre-commit/pre-commit-hooks - rev: "v4.3.0" + rev: "v4.4.0" hooks: - id: check-added-large-files - id: check-case-conflict @@ -30,109 +82,38 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace -# Upgrade old Python syntax -- repo: https://github.com/asottile/pyupgrade - rev: "v2.37.1" - hooks: - - id: pyupgrade - args: [--py36-plus] - -# Nicely sort includes -- repo: https://github.com/PyCQA/isort - rev: "5.10.1" - hooks: - - id: isort - -# Black, the code formatter, natively supports pre-commit -- repo: https://github.com/psf/black - rev: "22.6.0" # Keep in sync with blacken-docs - hooks: - - id: black - # Also code format the docs - repo: https://github.com/asottile/blacken-docs - rev: "v1.12.1" + rev: "1.14.0" hooks: - id: blacken-docs additional_dependencies: - - black==22.6.0 # keep in sync with black hook + - black==23.3.0 # keep in sync with black hook # Changes tabs to spaces - repo: https://github.com/Lucas-C/pre-commit-hooks - rev: "v1.3.0" + rev: "v1.5.1" hooks: - id: remove-tabs +# Avoid directional quotes - repo: https://github.com/sirosen/texthooks - rev: "0.3.1" + rev: "0.5.0" hooks: - id: fix-ligatures - id: fix-smartquotes -# Autoremoves unused imports -- repo: https://github.com/hadialqattan/pycln - rev: "v2.0.1" - hooks: - - id: pycln - stages: [manual] - # Checking for common mistakes - repo: https://github.com/pre-commit/pygrep-hooks - rev: "v1.9.0" + rev: "v1.10.0" hooks: - - id: python-check-blanket-noqa - - id: python-check-blanket-type-ignore - - id: python-no-log-warn - - id: python-use-type-annotations - id: rst-backticks - id: rst-directive-colons - id: rst-inline-touching-normal -# Automatically remove noqa that are not used -- repo: https://github.com/asottile/yesqa - rev: "v1.3.0" - hooks: - - id: yesqa - additional_dependencies: &flake8_dependencies - - flake8-bugbear - - pep8-naming - -# Flake8 also supports pre-commit natively (same author) -- repo: https://github.com/PyCQA/flake8 - rev: "4.0.1" - hooks: - - id: flake8 - exclude: ^(docs/.*|tools/.*)$ - additional_dependencies: *flake8_dependencies - -# PyLint has native support - not always usable, but works for us -- repo: https://github.com/PyCQA/pylint - rev: "v2.14.4" - hooks: - - id: pylint - files: ^pybind11 - -# CMake formatting -- repo: https://github.com/cheshirekow/cmake-format-precommit - rev: "v0.6.13" - hooks: - - id: cmake-format - additional_dependencies: [pyyaml] - types: [file] - files: (\.cmake|CMakeLists.txt)(.in)?$ - -# Check static types with mypy -- repo: https://github.com/pre-commit/mirrors-mypy - rev: "v0.961" - hooks: - - id: mypy - args: [] - exclude: ^(tests|docs)/ - additional_dependencies: [nox, rich] - # Checks the manifest for missing files (native support) - repo: https://github.com/mgedmin/check-manifest - rev: "0.48" + rev: "0.49" hooks: - id: check-manifest # This is a slow hook, so only run this if --hook-stage manual is passed @@ -140,16 +121,18 @@ repos: additional_dependencies: [cmake, ninja] # Check for spelling +# Use tools/codespell_ignore_lines_from_errors.py +# to rebuild .codespell-ignore-lines - repo: https://github.com/codespell-project/codespell - rev: "v2.1.0" + rev: "v2.2.5" hooks: - id: codespell exclude: ".supp$" - args: ["-L", "nd,ot,thist"] + args: ["-x.codespell-ignore-lines", "-Lccompiler"] # Check for common shell mistakes - repo: https://github.com/shellcheck-py/shellcheck-py - rev: "v0.8.0.4" + rev: "v0.9.0.5" hooks: - id: shellcheck @@ -162,9 +145,9 @@ repos: entry: PyBind|Numpy|Cmake|CCache|PyTest exclude: ^\.pre-commit-config.yaml$ -# Clang format the codebase automatically -- repo: https://github.com/pre-commit/mirrors-clang-format - rev: "v14.0.6" +# PyLint has native support - not always usable, but works for us +- repo: https://github.com/PyCQA/pylint + rev: "v3.0.0a6" hooks: - - id: clang-format - types_or: [c++, c, cuda] + - id: pylint + files: ^pybind11 diff --git a/pybind11/CMakeLists.txt b/pybind11/CMakeLists.txt index 3787982..87ec103 100644 --- a/pybind11/CMakeLists.txt +++ b/pybind11/CMakeLists.txt @@ -5,15 +5,15 @@ # All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) -# The `cmake_minimum_required(VERSION 3.4...3.22)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.22) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.22) + cmake_policy(VERSION 3.26) endif() # Avoid infinite recursion if tests include this as a subdirectory @@ -91,10 +91,16 @@ endif() option(PYBIND11_INSTALL "Install pybind11 header files?" ${PYBIND11_MASTER_PROJECT}) option(PYBIND11_TEST "Build pybind11 test suite?" ${PYBIND11_MASTER_PROJECT}) option(PYBIND11_NOPYTHON "Disable search for Python" OFF) +option(PYBIND11_SIMPLE_GIL_MANAGEMENT + "Use simpler GIL management logic that does not support disassociation" OFF) set(PYBIND11_INTERNALS_VERSION "" CACHE STRING "Override the ABI version, may be used to enable the unstable ABI.") +if(PYBIND11_SIMPLE_GIL_MANAGEMENT) + add_compile_definitions(PYBIND11_SIMPLE_GIL_MANAGEMENT) +endif() + cmake_dependent_option( USE_PYTHON_INCLUDE_DIR "Install pybind11 headers in Python include directory instead of default installation prefix" @@ -120,6 +126,9 @@ set(PYBIND11_HEADERS include/pybind11/complex.h include/pybind11/options.h include/pybind11/eigen.h + include/pybind11/eigen/common.h + include/pybind11/eigen/matrix.h + include/pybind11/eigen/tensor.h include/pybind11/embed.h include/pybind11/eval.h include/pybind11/gil.h @@ -131,7 +140,8 @@ set(PYBIND11_HEADERS include/pybind11/pytypes.h include/pybind11/stl.h include/pybind11/stl_bind.h - include/pybind11/stl/filesystem.h) + include/pybind11/stl/filesystem.h + include/pybind11/type_caster_pyobject_ptr.h) # Compare with grep and warn if mismatched if(PYBIND11_MASTER_PROJECT AND NOT CMAKE_VERSION VERSION_LESS 3.12) @@ -198,6 +208,9 @@ else() endif() include("${CMAKE_CURRENT_SOURCE_DIR}/tools/pybind11Common.cmake") +# https://github.com/jtojnar/cmake-snips/#concatenating-paths-when-building-pkg-config-files +# TODO: cmake 3.20 adds the cmake_path() function, which obsoletes this snippet +include("${CMAKE_CURRENT_SOURCE_DIR}/tools/JoinPaths.cmake") # Relative directory setting if(USE_PYTHON_INCLUDE_DIR AND DEFINED Python_INCLUDE_DIRS) @@ -262,6 +275,16 @@ if(PYBIND11_INSTALL) NAMESPACE "pybind11::" DESTINATION ${PYBIND11_CMAKECONFIG_INSTALL_DIR}) + # pkg-config support + if(NOT prefix_for_pc_file) + set(prefix_for_pc_file "${CMAKE_INSTALL_PREFIX}") + endif() + join_paths(includedir_for_pc_file "\${prefix}" "${CMAKE_INSTALL_INCLUDEDIR}") + configure_file("${CMAKE_CURRENT_SOURCE_DIR}/tools/pybind11.pc.in" + "${CMAKE_CURRENT_BINARY_DIR}/pybind11.pc" @ONLY) + install(FILES "${CMAKE_CURRENT_BINARY_DIR}/pybind11.pc" + DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/pkgconfig/") + # Uninstall target if(PYBIND11_MASTER_PROJECT) configure_file("${CMAKE_CURRENT_SOURCE_DIR}/tools/cmake_uninstall.cmake.in" diff --git a/pybind11/MANIFEST.in b/pybind11/MANIFEST.in index 033303a..7ce83c5 100644 --- a/pybind11/MANIFEST.in +++ b/pybind11/MANIFEST.in @@ -1,5 +1,6 @@ +prune tests recursive-include pybind11/include/pybind11 *.h recursive-include pybind11 *.py recursive-include pybind11 py.typed include pybind11/share/cmake/pybind11/*.cmake -include LICENSE README.rst pyproject.toml setup.py setup.cfg +include LICENSE README.rst SECURITY.md pyproject.toml setup.py setup.cfg diff --git a/pybind11/README.rst b/pybind11/README.rst index 3c75edb..80213a4 100644 --- a/pybind11/README.rst +++ b/pybind11/README.rst @@ -135,7 +135,7 @@ This project was created by `Wenzel Jakob `_. Significant features and/or improvements to the code were contributed by Jonas Adler, Lori A. Burns, Sylvain Corlay, Eric Cousineau, Aaron Gokaslan, Ralf Grosse-Kunstleve, Trent Houliston, Axel -Huebl, @hulucc, Yannick Jadoul, Sergey Lyskov Johan Mabille, Tomasz MiΔ…sko, +Huebl, @hulucc, Yannick Jadoul, Sergey Lyskov, Johan Mabille, Tomasz MiΔ…sko, Dean Moldovan, Ben Pritchard, Jason Rhinelander, Boris SchΓ€ling, Pim Schellart, Henry Schreiner, Ivan Smirnov, Boris Staletic, and Patrick Stewart. diff --git a/pybind11/SECURITY.md b/pybind11/SECURITY.md new file mode 100644 index 0000000..3d74611 --- /dev/null +++ b/pybind11/SECURITY.md @@ -0,0 +1,13 @@ +# Security Policy + +## Supported Versions + +Security updates are applied only to the latest release. + +## Reporting a Vulnerability + +If you have discovered a security vulnerability in this project, please report it privately. **Do not disclose it as a public issue.** This gives us time to work with you to fix the issue before public exposure, reducing the chance that the exploit will be used before a patch is released. + +Please disclose it at [security advisory](https://github.com/pybind/pybind11/security/advisories/new). + +This project is maintained by a team of volunteers on a reasonable-effort basis. As such, please give us at least 90 days to work on a fix before public exposure. diff --git a/pybind11/docs/advanced/cast/custom.rst b/pybind11/docs/advanced/cast/custom.rst index 1df4d3e..8138cac 100644 --- a/pybind11/docs/advanced/cast/custom.rst +++ b/pybind11/docs/advanced/cast/custom.rst @@ -38,7 +38,7 @@ type is explicitly allowed. .. code-block:: cpp - namespace pybind11 { namespace detail { + namespace PYBIND11_NAMESPACE { namespace detail { template <> struct type_caster { public: /** @@ -78,7 +78,7 @@ type is explicitly allowed. return PyLong_FromLong(src.long_value); } }; - }} // namespace pybind11::detail + }} // namespace PYBIND11_NAMESPACE::detail .. note:: diff --git a/pybind11/docs/advanced/cast/stl.rst b/pybind11/docs/advanced/cast/stl.rst index 109763f..03d49b2 100644 --- a/pybind11/docs/advanced/cast/stl.rst +++ b/pybind11/docs/advanced/cast/stl.rst @@ -42,7 +42,7 @@ types: .. code-block:: cpp // `boost::optional` as an example -- can be any `std::optional`-like container - namespace pybind11 { namespace detail { + namespace PYBIND11_NAMESPACE { namespace detail { template struct type_caster> : optional_caster> {}; }} @@ -54,7 +54,7 @@ for custom variant types: .. code-block:: cpp // `boost::variant` as an example -- can be any `std::variant`-like container - namespace pybind11 { namespace detail { + namespace PYBIND11_NAMESPACE { namespace detail { template struct type_caster> : variant_caster> {}; @@ -66,7 +66,7 @@ for custom variant types: return boost::apply_visitor(args...); } }; - }} // namespace pybind11::detail + }} // namespace PYBIND11_NAMESPACE::detail The ``visit_helper`` specialization is not required if your ``name::variant`` provides a ``name::visit()`` function. For any other function name, the specialization must be diff --git a/pybind11/docs/advanced/cast/strings.rst b/pybind11/docs/advanced/cast/strings.rst index e246c52..271716b 100644 --- a/pybind11/docs/advanced/cast/strings.rst +++ b/pybind11/docs/advanced/cast/strings.rst @@ -101,8 +101,11 @@ conversion has the same overhead as implicit conversion. m.def("str_output", []() { std::string s = "Send your r\xe9sum\xe9 to Alice in HR"; // Latin-1 - py::str py_s = PyUnicode_DecodeLatin1(s.data(), s.length()); - return py_s; + py::handle py_s = PyUnicode_DecodeLatin1(s.data(), s.length(), nullptr); + if (!py_s) { + throw py::error_already_set(); + } + return py::reinterpret_steal(py_s); } ); @@ -113,7 +116,8 @@ conversion has the same overhead as implicit conversion. The `Python C API `_ provides -several built-in codecs. +several built-in codecs. Note that these all return *new* references, so +use :cpp:func:`reinterpret_steal` when converting them to a :cpp:class:`str`. One could also use a third party encoding library such as libiconv to transcode diff --git a/pybind11/docs/advanced/classes.rst b/pybind11/docs/advanced/classes.rst index 49ddf5c..01a490b 100644 --- a/pybind11/docs/advanced/classes.rst +++ b/pybind11/docs/advanced/classes.rst @@ -1228,7 +1228,7 @@ whether a downcast is safe, you can proceed by specializing the std::string bark() const { return sound; } }; - namespace pybind11 { + namespace PYBIND11_NAMESPACE { template<> struct polymorphic_type_hook { static const void *get(const Pet *src, const std::type_info*& type) { // note that src may be nullptr @@ -1239,7 +1239,7 @@ whether a downcast is safe, you can proceed by specializing the return src; } }; - } // namespace pybind11 + } // namespace PYBIND11_NAMESPACE When pybind11 wants to convert a C++ pointer of type ``Base*`` to a Python object, it calls ``polymorphic_type_hook::get()`` to diff --git a/pybind11/docs/advanced/embedding.rst b/pybind11/docs/advanced/embedding.rst index dd980d4..e6a1686 100644 --- a/pybind11/docs/advanced/embedding.rst +++ b/pybind11/docs/advanced/embedding.rst @@ -18,7 +18,7 @@ information, see :doc:`/compiling`. .. code-block:: cmake - cmake_minimum_required(VERSION 3.4) + cmake_minimum_required(VERSION 3.5...3.26) project(example) find_package(pybind11 REQUIRED) # or `add_subdirectory(pybind11)` diff --git a/pybind11/docs/advanced/exceptions.rst b/pybind11/docs/advanced/exceptions.rst index 2211caf..53981dc 100644 --- a/pybind11/docs/advanced/exceptions.rst +++ b/pybind11/docs/advanced/exceptions.rst @@ -177,9 +177,12 @@ section. may be explicitly (re-)thrown to delegate it to the other, previously-declared existing exception translators. - Note that ``libc++`` and ``libstdc++`` `behave differently `_ - with ``-fvisibility=hidden``. Therefore exceptions that are used across ABI boundaries need to be explicitly exported, as exercised in ``tests/test_exceptions.h``. - See also: "Problems with C++ exceptions" under `GCC Wiki `_. + Note that ``libc++`` and ``libstdc++`` `behave differently under macOS + `_ + with ``-fvisibility=hidden``. Therefore exceptions that are used across ABI + boundaries need to be explicitly exported, as exercised in + ``tests/test_exceptions.h``. See also: + "Problems with C++ exceptions" under `GCC Wiki `_. Local vs Global Exception Translators diff --git a/pybind11/docs/advanced/misc.rst b/pybind11/docs/advanced/misc.rst index edab15f..805ec83 100644 --- a/pybind11/docs/advanced/misc.rst +++ b/pybind11/docs/advanced/misc.rst @@ -39,15 +39,42 @@ The ``PYBIND11_MAKE_OPAQUE`` macro does *not* require the above workarounds. Global Interpreter Lock (GIL) ============================= -When calling a C++ function from Python, the GIL is always held. +The Python C API dictates that the Global Interpreter Lock (GIL) must always +be held by the current thread to safely access Python objects. As a result, +when Python calls into C++ via pybind11 the GIL must be held, and pybind11 +will never implicitly release the GIL. + +.. code-block:: cpp + + void my_function() { + /* GIL is held when this function is called from Python */ + } + + PYBIND11_MODULE(example, m) { + m.def("my_function", &my_function); + } + +pybind11 will ensure that the GIL is held when it knows that it is calling +Python code. For example, if a Python callback is passed to C++ code via +``std::function``, when C++ code calls the function the built-in wrapper +will acquire the GIL before calling the Python callback. Similarly, the +``PYBIND11_OVERRIDE`` family of macros will acquire the GIL before calling +back into Python. + +When writing C++ code that is called from other C++ code, if that code accesses +Python state, it must explicitly acquire and release the GIL. + The classes :class:`gil_scoped_release` and :class:`gil_scoped_acquire` can be used to acquire and release the global interpreter lock in the body of a C++ function call. In this way, long-running C++ code can be parallelized using -multiple Python threads. Taking :ref:`overriding_virtuals` as an example, this +multiple Python threads, **but great care must be taken** when any +:class:`gil_scoped_release` appear: if there is any way that the C++ code +can access Python objects, :class:`gil_scoped_acquire` should be used to +reacquire the GIL. Taking :ref:`overriding_virtuals` as an example, this could be realized as follows (important changes highlighted): .. code-block:: cpp - :emphasize-lines: 8,9,31,32 + :emphasize-lines: 8,30,31 class PyAnimal : public Animal { public: @@ -56,9 +83,7 @@ could be realized as follows (important changes highlighted): /* Trampoline (need one for each virtual function) */ std::string go(int n_times) { - /* Acquire GIL before calling Python code */ - py::gil_scoped_acquire acquire; - + /* PYBIND11_OVERRIDE_PURE will acquire the GIL before accessing Python state */ PYBIND11_OVERRIDE_PURE( std::string, /* Return type */ Animal, /* Parent class */ @@ -78,7 +103,8 @@ could be realized as follows (important changes highlighted): .def(py::init<>()); m.def("call_go", [](Animal *animal) -> std::string { - /* Release GIL before calling into (potentially long-running) C++ code */ + // GIL is held when called from Python code. Release GIL before + // calling into (potentially long-running) C++ code py::gil_scoped_release release; return call_go(animal); }); @@ -92,6 +118,34 @@ The ``call_go`` wrapper can also be simplified using the ``call_guard`` policy m.def("call_go", &call_go, py::call_guard()); +Common Sources Of Global Interpreter Lock Errors +================================================================== + +Failing to properly hold the Global Interpreter Lock (GIL) is one of the +more common sources of bugs within code that uses pybind11. If you are +running into GIL related errors, we highly recommend you consult the +following checklist. + +- Do you have any global variables that are pybind11 objects or invoke + pybind11 functions in either their constructor or destructor? You are generally + not allowed to invoke any Python function in a global static context. We recommend + using lazy initialization and then intentionally leaking at the end of the program. + +- Do you have any pybind11 objects that are members of other C++ structures? One + commonly overlooked requirement is that pybind11 objects have to increase their reference count + whenever their copy constructor is called. Thus, you need to be holding the GIL to invoke + the copy constructor of any C++ class that has a pybind11 member. This can sometimes be very + tricky to track for complicated programs Think carefully when you make a pybind11 object + a member in another struct. + +- C++ destructors that invoke Python functions can be particularly troublesome as + destructors can sometimes get invoked in weird and unexpected circumstances as a result + of exceptions. + +- You should try running your code in a debug build. That will enable additional assertions + within pybind11 that will throw exceptions on certain GIL handling errors + (reference counting operations). + Binding sequence data types, iterators, the slicing protocol, etc. ================================================================== @@ -298,6 +352,15 @@ The class ``options`` allows you to selectively suppress auto-generated signatur m.def("add", [](int a, int b) { return a + b; }, "A function which adds two numbers"); } +pybind11 also appends all members of an enum to the resulting enum docstring. +This default behavior can be disabled by using the ``disable_enum_members_docstring()`` +function of the ``options`` class. + +With ``disable_user_defined_docstrings()`` all user defined docstrings of +``module_::def()``, ``class_::def()`` and ``enum_()`` are disabled, but the +function signatures and enum members are included in the docstring, unless they +are disabled separately. + Note that changes to the settings affect only function bindings created during the lifetime of the ``options`` instance. When it goes out of scope at the end of the module's init function, the default settings are restored to prevent unwanted side effects. diff --git a/pybind11/docs/advanced/pycpp/numpy.rst b/pybind11/docs/advanced/pycpp/numpy.rst index b6ef019..07c9693 100644 --- a/pybind11/docs/advanced/pycpp/numpy.rst +++ b/pybind11/docs/advanced/pycpp/numpy.rst @@ -433,7 +433,7 @@ following: { 2, 4 }, // shape (rows, cols) { sizeof(uint8_t) * 4, sizeof(uint8_t) } // strides in bytes ); - }) + }); This approach is meant for providing a ``memoryview`` for a C/C++ buffer not managed by Python. The user is responsible for managing the lifetime of the @@ -449,7 +449,7 @@ We can also use ``memoryview::from_memory`` for a simple 1D contiguous buffer: buffer, // buffer pointer sizeof(uint8_t) * 8 // buffer size ); - }) + }); .. versionchanged:: 2.6 ``memoryview::from_memory`` added. diff --git a/pybind11/docs/advanced/smart_ptrs.rst b/pybind11/docs/advanced/smart_ptrs.rst index 5a22201..3c40ce1 100644 --- a/pybind11/docs/advanced/smart_ptrs.rst +++ b/pybind11/docs/advanced/smart_ptrs.rst @@ -157,7 +157,7 @@ specialized: PYBIND11_DECLARE_HOLDER_TYPE(T, SmartPtr); // Only needed if the type's `.get()` goes by another name - namespace pybind11 { namespace detail { + namespace PYBIND11_NAMESPACE { namespace detail { template struct holder_helper> { // <-- specialization static const T *get(const SmartPtr &p) { return p.getPointer(); } diff --git a/pybind11/docs/changelog.rst b/pybind11/docs/changelog.rst index b926b27..add3fd6 100644 --- a/pybind11/docs/changelog.rst +++ b/pybind11/docs/changelog.rst @@ -9,6 +9,364 @@ Starting with version 1.8.0, pybind11 releases use a `semantic versioning Changes will be added here periodically from the "Suggested changelog entry" block in pull request descriptions. + +Version 2.11.1 (July 17, 2023) +----------------------------- + +Changes: + +* ``PYBIND11_NO_ASSERT_GIL_HELD_INCREF_DECREF`` is now provided as an option + for disabling the default-on ``PyGILState_Check()``'s in + ``pybind11::handle``'s ``inc_ref()`` & ``dec_ref()``. + `#4753 `_ + +* ``PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF`` was disabled for PyPy in general + (not just PyPy Windows). + `#4751 `_ + + +Version 2.11.0 (July 14, 2023) +----------------------------- + +New features: + +* The newly added ``pybind11::detail::is_move_constructible`` trait can be + specialized for cases in which ``std::is_move_constructible`` does not work + as needed. This is very similar to the long-established + ``pybind11::detail::is_copy_constructible``. + `#4631 `_ + +* Introduce ``recursive_container_traits``. + `#4623 `_ + +* ``pybind11/type_caster_pyobject_ptr.h`` was added to support automatic + wrapping of APIs that make use of ``PyObject *``. This header needs to + included explicitly (i.e. it is not included implicitly + with ``pybind/pybind11.h``). + `#4601 `_ + +* ``format_descriptor<>`` & ``npy_format_descriptor<>`` ``PyObject *`` + specializations were added. The latter enables ``py::array_t`` + to/from-python conversions. + `#4674 `_ + +* ``buffer_info`` gained an ``item_type_is_equivalent_to()`` member + function. + `#4674 `_ + +* The ``capsule`` API gained a user-friendly constructor + (``py::capsule(ptr, "name", dtor)``). + `#4720 `_ + +Changes: + +* ``PyGILState_Check()``'s in ``pybind11::handle``'s ``inc_ref()`` & + ``dec_ref()`` are now enabled by default again. + `#4246 `_ + +* ``py::initialize_interpreter()`` using ``PyConfig_InitPythonConfig()`` + instead of ``PyConfig_InitIsolatedConfig()``, to obtain complete + ``sys.path``. + `#4473 `_ + +* Cast errors now always include Python type information, even if + ``PYBIND11_DETAILED_ERROR_MESSAGES`` is not defined. This increases binary + sizes slightly (~1.5%) but the error messages are much more informative. + `#4463 `_ + +* The docstring generation for the ``std::array``-list caster was fixed. + Previously, signatures included the size of the list in a non-standard, + non-spec compliant way. The new format conforms to PEP 593. + **Tooling for processing the docstrings may need to be updated accordingly.** + `#4679 `_ + +* Setter return values (which are inaccessible for all practical purposes) are + no longer converted to Python (only to be discarded). + `#4621 `_ + +* Allow lambda specified to function definition to be ``noexcept(true)`` + in C++17. + `#4593 `_ + +* Get rid of recursive template instantiations for concatenating type + signatures on C++17 and higher. + `#4587 `_ + +* Compatibility with Python 3.12 (beta). Note that the minimum pybind11 + ABI version for Python 3.12 is version 5. (The default ABI version + for Python versions up to and including 3.11 is still version 4.). + `#4570 `_ + +* With ``PYBIND11_INTERNALS_VERSION 5`` (default for Python 3.12+), MSVC builds + use ``std::hash`` and ``std::equal_to`` + instead of string-based type comparisons. This resolves issues when binding + types defined in the unnamed namespace. + `#4319 `_ + +* Python exception ``__notes__`` (introduced with Python 3.11) are now added to + the ``error_already_set::what()`` output. + `#4678 `_ + +Build system improvements: + +* CMake 3.27 support was added, CMake 3.4 support was dropped. + FindPython will be used if ``FindPythonInterp`` is not present. + `#4719 `_ + +* Update clang-tidy to 15 in CI. + `#4387 `_ + +* Moved the linting framework over to Ruff. + `#4483 `_ + +* Skip ``lto`` checks and target generation when + ``CMAKE_INTERPROCEDURAL_OPTIMIZATION`` is defined. + `#4643 `_ + +* No longer inject ``-stdlib=libc++``, not needed for modern Pythons + (macOS 10.9+). + `#4639 `_ + +* PyPy 3.10 support was added, PyPy 3.7 support was dropped. + `#4728 `_ + +* Testing with Python 3.12 beta releases was added. + `#4713 `_ + + +Version 2.10.4 (Mar 16, 2023) +----------------------------- + +Changes: + +* ``python3 -m pybind11`` gained a ``--version`` option (prints the version and + exits). + `#4526 `_ + +Bug Fixes: + +* Fix a warning when pydebug is enabled on Python 3.11. + `#4461 `_ + +* Ensure ``gil_scoped_release`` RAII is non-copyable. + `#4490 `_ + +* Ensure the tests dir does not show up with new versions of setuptools. + `#4510 `_ + +* Better stacklevel for a warning in setuptools helpers. + `#4516 `_ + +Version 2.10.3 (Jan 3, 2023) +---------------------------- + +Changes: + +* Temporarily made our GIL status assertions (added in 2.10.2) disabled by + default (re-enable manually by defining + ``PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF``, will be enabled in 2.11). + `#4432 `_ + +* Improved error messages when ``inc_ref``/``dec_ref`` are called with an + invalid GIL state. + `#4427 `_ + `#4436 `_ + +Bug Fixes: + +* Some minor touchups found by static analyzers. + `#4440 `_ + + +Version 2.10.2 (Dec 20, 2022) +----------------------------- + +Changes: + +* ``scoped_interpreter`` constructor taking ``PyConfig``. + `#4330 `_ + +* ``pybind11/eigen/tensor.h`` adds converters to and from ``Eigen::Tensor`` and + ``Eigen::TensorMap``. + `#4201 `_ + +* ``PyGILState_Check()``'s were integrated to ``pybind11::handle`` + ``inc_ref()`` & ``dec_ref()``. The added GIL checks are guarded by + ``PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF``, which is the default only if + ``NDEBUG`` is not defined. (Made non-default in 2.10.3, will be active in 2.11) + `#4246 `_ + +* Add option for enable/disable enum members in docstring. + `#2768 `_ + +* Fixed typing of ``KeysView``, ``ValuesView`` and ``ItemsView`` in ``bind_map``. + `#4353 `_ + +Bug fixes: + +* Bug fix affecting only Python 3.6 under very specific, uncommon conditions: + move ``PyEval_InitThreads()`` call to the correct location. + `#4350 `_ + +* Fix segfault bug when passing foreign native functions to functional.h. + `#4254 `_ + +Build system improvements: + +* Support setting PYTHON_LIBRARIES manually for Windows ARM cross-compilation + (classic mode). + `#4406 `_ + +* Extend IPO/LTO detection for ICX (a.k.a IntelLLVM) compiler. + `#4402 `_ + +* Allow calling ``find_package(pybind11 CONFIG)`` multiple times from separate + directories in the same CMake project and properly link Python (new mode). + `#4401 `_ + +* ``multiprocessing_set_spawn`` in pytest fixture for added safety. + `#4377 `_ + +* Fixed a bug in two pybind11/tools cmake scripts causing "Unknown arguments specified" errors. + `#4327 `_ + + + +Version 2.10.1 (Oct 31, 2022) +----------------------------- + +This is the first version to fully support embedding the newly released Python 3.11. + +Changes: + +* Allow ``pybind11::capsule`` constructor to take null destructor pointers. + `#4221 `_ + +* ``embed.h`` was changed so that ``PYTHONPATH`` is used also with Python 3.11 + (established behavior). + `#4119 `_ + +* A ``PYBIND11_SIMPLE_GIL_MANAGEMENT`` option was added (cmake, C++ define), + along with many additional tests in ``test_gil_scoped.py``. The option may be + useful to try when debugging GIL-related issues, to determine if the more + complex default implementation is or is not to blame. See #4216 for + background. WARNING: Please be careful to not create ODR violations when + using the option: everything that is linked together with mutual symbol + visibility needs to be rebuilt. + `#4216 `_ + +* ``PYBIND11_EXPORT_EXCEPTION`` was made non-empty only under macOS. This makes + Linux builds safer, and enables the removal of warning suppression pragmas for + Windows. + `#4298 `_ + +Bug fixes: + +* Fixed a bug where ``UnicodeDecodeError`` was not propagated from various + ``py::str`` ctors when decoding surrogate utf characters. + `#4294 `_ + +* Revert perfect forwarding for ``make_iterator``. This broke at least one + valid use case. May revisit later. + `#4234 `_ + +* Fix support for safe casts to ``void*`` (regression in 2.10.0). + `#4275 `_ + +* Fix ``char8_t`` support (regression in 2.9). + `#4278 `_ + +* Unicode surrogate character in Python exception message leads to process + termination in ``error_already_set::what()``. + `#4297 `_ + +* Fix MSVC 2019 v.1924 & C++14 mode error for ``overload_cast``. + `#4188 `_ + +* Make augmented assignment operators non-const for the object-api. Behavior + was previously broken for augmented assignment operators. + `#4065 `_ + +* Add proper error checking to C++ bindings for Python list append and insert. + `#4208 `_ + +* Work-around for Nvidia's CUDA nvcc compiler in versions 11.4.0 - 11.8.0. + `#4220 `_ + +* A workaround for PyPy was added in the ``py::error_already_set`` + implementation, related to PR `#1895 `_ + released with v2.10.0. + `#4079 `_ + +* Fixed compiler errors when C++23 ``std::forward_like`` is available. + `#4136 `_ + +* Properly raise exceptions in contains methods (like when an object in unhashable). + `#4209 `_ + +* Further improve another error in exception handling. + `#4232 `_ + +* ``get_local_internals()`` was made compatible with + ``finalize_interpreter()``, fixing potential freezes during interpreter + finalization. + `#4192 `_ + +Performance and style: + +* Reserve space in set and STL map casters if possible. This will prevent + unnecessary rehashing / resizing by knowing the number of keys ahead of time + for Python to C++ casting. This improvement will greatly speed up the casting + of large unordered maps and sets. + `#4194 `_ + +* GIL RAII scopes are non-copyable to avoid potential bugs. + `#4183 `_ + +* Explicitly default all relevant ctors for pytypes in the ``PYBIND11_OBJECT`` + macros and enforce the clang-tidy checks ``modernize-use-equals-default`` in + macros as well. + `#4017 `_ + +* Optimize iterator advancement in C++ bindings. + `#4237 `_ + +* Use the modern ``PyObject_GenericGetDict`` and ``PyObject_GenericSetDict`` + for handling dynamic attribute dictionaries. + `#4106 `_ + +* Document that users should use ``PYBIND11_NAMESPACE`` instead of using ``pybind11`` when + opening namespaces. Using namespace declarations and namespace qualification + remain the same as ``pybind11``. This is done to ensure consistent symbol + visibility. + `#4098 `_ + +* Mark ``detail::forward_like`` as constexpr. + `#4147 `_ + +* Optimize unpacking_collector when processing ``arg_v`` arguments. + `#4219 `_ + +* Optimize casting C++ object to ``None``. + `#4269 `_ + + +Build system improvements: + +* CMake: revert overwrite behavior, now opt-in with ``PYBIND11_PYTHONLIBS_OVERRWRITE OFF``. + `#4195 `_ + +* Include a pkg-config file when installing pybind11, such as in the Python + package. + `#4077 `_ + +* Avoid stripping debug symbols when ``CMAKE_BUILD_TYPE`` is set to ``DEBUG`` + instead of ``Debug``. + `#4078 `_ + +* Followup to `#3948 `_, fixing vcpkg again. + `#4123 `_ + Version 2.10.0 (Jul 15, 2022) ----------------------------- diff --git a/pybind11/docs/classes.rst b/pybind11/docs/classes.rst index c0c5313..4f2167d 100644 --- a/pybind11/docs/classes.rst +++ b/pybind11/docs/classes.rst @@ -58,6 +58,16 @@ interactive Python session demonstrating this example is shown below: Static member functions can be bound in the same way using :func:`class_::def_static`. +.. note:: + + Binding C++ types in unnamed namespaces (also known as anonymous namespaces) + works reliably on many platforms, but not all. The `XFAIL_CONDITION` in + tests/test_unnamed_namespace_a.py encodes the currently known conditions. + For background see `#4319 `_. + If portability is a concern, it is therefore not recommended to bind C++ + types in unnamed namespaces. It will be safest to manually pick unique + namespace names. + Keyword and default arguments ============================= It is possible to specify keyword and default arguments using the syntax @@ -539,3 +549,7 @@ The ``name`` property returns the name of the enum value as a unicode string. ... By default, these are omitted to conserve space. + +.. warning:: + + Contrary to Python customs, enum values from the wrappers should not be compared using ``is``, but with ``==`` (see `#1177 `_ for background). diff --git a/pybind11/docs/compiling.rst b/pybind11/docs/compiling.rst index 2b543be..1fd098b 100644 --- a/pybind11/docs/compiling.rst +++ b/pybind11/docs/compiling.rst @@ -241,7 +241,7 @@ extension module can be created with just a few lines of code: .. code-block:: cmake - cmake_minimum_required(VERSION 3.4...3.18) + cmake_minimum_required(VERSION 3.5...3.26) project(example LANGUAGES CXX) add_subdirectory(pybind11) @@ -261,6 +261,9 @@ PyPI integration, can be found in the [cmake_example]_ repository. .. versionchanged:: 2.6 CMake 3.4+ is required. +.. versionchanged:: 2.11 + CMake 3.5+ is required. + Further information can be found at :doc:`cmake/index`. pybind11_add_module @@ -495,7 +498,7 @@ You can use these targets to build complex applications. For example, the .. code-block:: cmake - cmake_minimum_required(VERSION 3.4) + cmake_minimum_required(VERSION 3.5...3.26) project(example LANGUAGES CXX) find_package(pybind11 REQUIRED) # or add_subdirectory(pybind11) @@ -553,7 +556,7 @@ information about usage in C++, see :doc:`/advanced/embedding`. .. code-block:: cmake - cmake_minimum_required(VERSION 3.4...3.18) + cmake_minimum_required(VERSION 3.5...3.26) project(example LANGUAGES CXX) find_package(pybind11 REQUIRED) # or add_subdirectory(pybind11) diff --git a/pybind11/docs/conf.py b/pybind11/docs/conf.py index 2da6773..6e24751 100644 --- a/pybind11/docs/conf.py +++ b/pybind11/docs/conf.py @@ -353,12 +353,11 @@ def prepare(app): f.write(contents) -def clean_up(app, exception): +def clean_up(app, exception): # noqa: ARG001 (DIR / "readme.rst").unlink() def setup(app): - # Add hook for building doxygen xml when needed app.connect("builder-inited", generate_doxygen_xml) diff --git a/pybind11/docs/faq.rst b/pybind11/docs/faq.rst index 28498e7..1eb00ef 100644 --- a/pybind11/docs/faq.rst +++ b/pybind11/docs/faq.rst @@ -284,7 +284,8 @@ There are three possible solutions: COMPONENTS Interpreter Development)`` on modern CMake (3.12+, 3.15+ better, 3.18.2+ best). Pybind11 in these cases uses the new CMake FindPython instead of the old, deprecated search tools, and these modules are much better at - finding the correct Python. + finding the correct Python. If FindPythonLibs/Interp are not available + (CMake 3.27+), then this will be ignored and FindPython will be used. 3. Set ``PYBIND11_NOPYTHON`` to ``TRUE``. Pybind11 will not search for Python. However, you will have to use the target-based system, and do more setup yourself, because it does not know about or include things that depend on diff --git a/pybind11/docs/release.rst b/pybind11/docs/release.rst index e761cdf..4950c3b 100644 --- a/pybind11/docs/release.rst +++ b/pybind11/docs/release.rst @@ -33,10 +33,12 @@ If you don't have nox, you should either use ``pipx run nox`` instead, or use - Run ``nox -s tests_packaging`` to ensure this was done correctly. - Ensure that all the information in ``setup.cfg`` is up-to-date, like supported Python versions. - - Add release date in ``docs/changelog.rst``. - - Check to make sure - `needs-changelog `_ - issues are entered in the changelog (clear the label when done). + - Add release date in ``docs/changelog.rst`` and integrate the output of + ``nox -s make_changelog``. + - Note that the ``make_changelog`` command inspects + `needs changelog `_. + - Manually clear the ``needs changelog`` labels using the GitHub web + interface (very easy: start by clicking the link above). - ``git add`` and ``git commit``, ``git push``. **Ensure CI passes**. (If it fails due to a known flake issue, either ignore or restart CI.) - Add a release branch if this is a new minor version, or update the existing release branch if it is a patch version diff --git a/pybind11/docs/upgrade.rst b/pybind11/docs/upgrade.rst index 6a9db2d..b13d21f 100644 --- a/pybind11/docs/upgrade.rst +++ b/pybind11/docs/upgrade.rst @@ -8,6 +8,20 @@ to a new version. But it goes into more detail. This includes things like deprecated APIs and their replacements, build system changes, general code modernization and other useful information. +.. _upgrade-guide-2.11: + +v2.11 +===== + +* The minimum version of CMake is now 3.5. A future version will likely move to + requiring something like CMake 3.15. Note that CMake 3.27 is removing the + long-deprecated support for ``FindPythonInterp`` if you set 3.27 as the + minimum or maximum supported version. To prepare for that future, CMake 3.15+ + using ``FindPython`` or setting ``PYBIND11_FINDPYTHON`` is highly recommended, + otherwise pybind11 will automatically switch to using ``FindPython`` if + ``FindPythonInterp`` is not available. + + .. _upgrade-guide-2.9: v2.9 diff --git a/pybind11/include/pybind11/attr.h b/pybind11/include/pybind11/attr.h index db7cd8e..1044db9 100644 --- a/pybind11/include/pybind11/attr.h +++ b/pybind11/include/pybind11/attr.h @@ -26,6 +26,9 @@ struct is_method { explicit is_method(const handle &c) : class_(c) {} }; +/// Annotation for setters +struct is_setter {}; + /// Annotation for operators struct is_operator {}; @@ -188,8 +191,8 @@ struct argument_record { struct function_record { function_record() : is_constructor(false), is_new_style_constructor(false), is_stateless(false), - is_operator(false), is_method(false), has_args(false), has_kwargs(false), - prepend(false) {} + is_operator(false), is_method(false), is_setter(false), has_args(false), + has_kwargs(false), prepend(false) {} /// Function name char *name = nullptr; /* why no C++ strings? They generate heavier code.. */ @@ -230,6 +233,9 @@ struct function_record { /// True if this is a method bool is_method : 1; + /// True if this is a setter + bool is_setter : 1; + /// True if the function has a '*args' argument bool has_args : 1; @@ -399,7 +405,7 @@ struct process_attribute : process_attribute_default { template <> struct process_attribute : process_attribute_default { static void init(const char *d, function_record *r) { r->doc = const_cast(d); } - static void init(const char *d, type_record *r) { r->doc = const_cast(d); } + static void init(const char *d, type_record *r) { r->doc = d; } }; template <> struct process_attribute : process_attribute {}; @@ -426,6 +432,12 @@ struct process_attribute : process_attribute_default { } }; +/// Process an attribute which indicates that this function is a setter +template <> +struct process_attribute : process_attribute_default { + static void init(const is_setter &, function_record *r) { r->is_setter = true; } +}; + /// Process an attribute which indicates the parent scope of a method template <> struct process_attribute : process_attribute_default { diff --git a/pybind11/include/pybind11/buffer_info.h b/pybind11/include/pybind11/buffer_info.h index 06120d5..b99ee8b 100644 --- a/pybind11/include/pybind11/buffer_info.h +++ b/pybind11/include/pybind11/buffer_info.h @@ -37,6 +37,9 @@ inline std::vector f_strides(const std::vector &shape, ssize_t return strides; } +template +struct compare_buffer_info; + PYBIND11_NAMESPACE_END(detail) /// Information record describing a Python buffer object @@ -150,6 +153,17 @@ struct buffer_info { Py_buffer *view() const { return m_view; } Py_buffer *&view() { return m_view; } + /* True if the buffer item type is equivalent to `T`. */ + // To define "equivalent" by example: + // `buffer_info::item_type_is_equivalent_to(b)` and + // `buffer_info::item_type_is_equivalent_to(b)` may both be true + // on some platforms, but `int` and `unsigned` will never be equivalent. + // For the ground truth, please inspect `detail::compare_buffer_info<>`. + template + bool item_type_is_equivalent_to() const { + return detail::compare_buffer_info::compare(*this); + } + private: struct private_ctr_tag {}; @@ -170,9 +184,10 @@ struct buffer_info { PYBIND11_NAMESPACE_BEGIN(detail) -template +template struct compare_buffer_info { static bool compare(const buffer_info &b) { + // NOLINTNEXTLINE(bugprone-sizeof-expression) Needed for `PyObject *` return b.format == format_descriptor::format() && b.itemsize == (ssize_t) sizeof(T); } }; diff --git a/pybind11/include/pybind11/cast.h b/pybind11/include/pybind11/cast.h index a0e3228..db39341 100644 --- a/pybind11/include/pybind11/cast.h +++ b/pybind11/include/pybind11/cast.h @@ -29,6 +29,9 @@ #include PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +PYBIND11_WARNING_DISABLE_MSVC(4127) + PYBIND11_NAMESPACE_BEGIN(detail) template @@ -88,7 +91,8 @@ public: template >::value, \ - int> = 0> \ + int> \ + = 0> \ static ::pybind11::handle cast( \ T_ *src, ::pybind11::return_value_policy policy, ::pybind11::handle parent) { \ if (!src) \ @@ -248,7 +252,7 @@ struct void_caster { return false; } static handle cast(T, return_value_policy /* policy */, handle /* parent */) { - return none().inc_ref(); + return none().release(); } PYBIND11_TYPE_CASTER(T, const_name("None")); }; @@ -291,7 +295,7 @@ class type_caster : public type_caster { if (ptr) { return capsule(ptr).release(); } - return none().inc_ref(); + return none().release(); } template @@ -389,7 +393,7 @@ struct string_caster { // For UTF-8 we avoid the need for a temporary `bytes` object by using // `PyUnicode_AsUTF8AndSize`. - if (PYBIND11_SILENCE_MSVC_C4127(UTF_N == 8)) { + if (UTF_N == 8) { Py_ssize_t size = -1; const auto *buffer = reinterpret_cast(PyUnicode_AsUTF8AndSize(load_src.ptr(), &size)); @@ -416,7 +420,7 @@ struct string_caster { = reinterpret_cast(PYBIND11_BYTES_AS_STRING(utfNbytes.ptr())); size_t length = (size_t) PYBIND11_BYTES_SIZE(utfNbytes.ptr()) / sizeof(CharT); // Skip BOM for UTF-16/32 - if (PYBIND11_SILENCE_MSVC_C4127(UTF_N > 8)) { + if (UTF_N > 8) { buffer++; length--; } @@ -537,7 +541,7 @@ struct type_caster::value>> { static handle cast(const CharT *src, return_value_policy policy, handle parent) { if (src == nullptr) { - return pybind11::none().inc_ref(); + return pybind11::none().release(); } return StringCaster::cast(StringType(src), policy, parent); } @@ -572,7 +576,7 @@ struct type_caster::value>> { // figure out how long the first encoded character is in bytes to distinguish between these // two errors. We also allow want to allow unicode characters U+0080 through U+00FF, as // those can fit into a single char value. - if (PYBIND11_SILENCE_MSVC_C4127(StringCaster::UTF_N == 8) && str_len > 1 && str_len <= 4) { + if (StringCaster::UTF_N == 8 && str_len > 1 && str_len <= 4) { auto v0 = static_cast(value[0]); // low bits only: 0-127 // 0b110xxxxx - start of 2-byte sequence @@ -598,7 +602,7 @@ struct type_caster::value>> { // UTF-16 is much easier: we can only have a surrogate pair for values above U+FFFF, thus a // surrogate pair with total length 2 instantly indicates a range error (but not a "your // string was too long" error). - else if (PYBIND11_SILENCE_MSVC_C4127(StringCaster::UTF_N == 16) && str_len == 2) { + else if (StringCaster::UTF_N == 16 && str_len == 2) { one_char = static_cast(value[0]); if (one_char >= 0xD800 && one_char < 0xE000) { throw value_error("Character code point not in range(0x10000)"); @@ -960,7 +964,7 @@ struct move_always< enable_if_t< all_of, negation>, - std::is_move_constructible, + is_move_constructible, std::is_same>().operator T &()), T &>>::value>> : std::true_type {}; template @@ -971,7 +975,7 @@ struct move_if_unreferenced< enable_if_t< all_of, negation>, - std::is_move_constructible, + is_move_constructible, std::is_same>().operator T &()), T &>>::value>> : std::true_type {}; template @@ -1013,11 +1017,14 @@ type_caster &load_type(type_caster &conv, const handle &ha "Internal error: type_caster should only be used for C++ types"); if (!conv.load(handle, true)) { #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) - throw cast_error("Unable to cast Python instance to C++ type (#define " - "PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); + throw cast_error( + "Unable to cast Python instance of type " + + str(type::handle_of(handle)).cast() + + " to C++ type '?' (#define " + "PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); #else throw cast_error("Unable to cast Python instance of type " - + (std::string) str(type::handle_of(handle)) + " to C++ type '" + + str(type::handle_of(handle)).cast() + " to C++ type '" + type_id() + "'"); #endif } @@ -1034,7 +1041,11 @@ make_caster load_type(const handle &handle) { PYBIND11_NAMESPACE_END(detail) // pytype -> C++ type -template ::value, int> = 0> +template ::value + && !detail::is_same_ignoring_cvref::value, + int> + = 0> T cast(const handle &handle) { using namespace detail; static_assert(!cast_is_temporary_value_reference::value, @@ -1048,6 +1059,34 @@ T cast(const handle &handle) { return T(reinterpret_borrow(handle)); } +// Note that `cast(obj)` increments the reference count of `obj`. +// This is necessary for the case that `obj` is a temporary, and could +// not possibly be different, given +// 1. the established convention that the passed `handle` is borrowed, and +// 2. we don't want to force all generic code using `cast()` to special-case +// handling of `T` = `PyObject *` (to increment the reference count there). +// It is the responsibility of the caller to ensure that the reference count +// is decremented. +template ::value + && detail::is_same_ignoring_cvref::value, + int> + = 0> +T cast(Handle &&handle) { + return handle.inc_ref().ptr(); +} +// To optimize way an inc_ref/dec_ref cycle: +template ::value + && detail::is_same_ignoring_cvref::value, + int> + = 0> +T cast(Object &&obj) { + return obj.release().ptr(); +} + // C++ type -> py::object template ::value, int> = 0> object cast(T &&value, @@ -1081,12 +1120,13 @@ detail::enable_if_t::value, T> move(object &&obj) { if (obj.ref_count() > 1) { #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) throw cast_error( - "Unable to cast Python instance to C++ rvalue: instance has multiple references" - " (#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); + "Unable to cast Python " + str(type::handle_of(obj)).cast() + + " instance to C++ rvalue: instance has multiple references" + " (#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); #else - throw cast_error("Unable to move from Python " + (std::string) str(type::handle_of(obj)) - + " instance to C++ " + type_id() - + " instance: instance has multiple references"); + throw cast_error("Unable to move from Python " + + str(type::handle_of(obj)).cast() + " instance to C++ " + + type_id() + " instance: instance has multiple references"); #endif } @@ -1179,11 +1219,9 @@ enable_if_t::value, T> cast_safe(object &&) pybind11_fail("Internal error: cast_safe fallback invoked"); } template -enable_if_t>::value, void> cast_safe(object &&) {} +enable_if_t::value, void> cast_safe(object &&) {} template -enable_if_t, - std::is_same>>::value, - T> +enable_if_t, std::is_void>::value, T> cast_safe(object &&o) { return pybind11::cast(std::move(o)); } @@ -1193,9 +1231,10 @@ PYBIND11_NAMESPACE_END(detail) // The overloads could coexist, i.e. the #if is not strictly speaking needed, // but it is an easy minor optimization. #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) -inline cast_error cast_error_unable_to_convert_call_arg() { - return cast_error("Unable to convert call argument to Python object (#define " - "PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); +inline cast_error cast_error_unable_to_convert_call_arg(const std::string &name) { + return cast_error("Unable to convert call argument '" + name + + "' to Python object (#define " + "PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); } #else inline cast_error cast_error_unable_to_convert_call_arg(const std::string &name, @@ -1218,7 +1257,7 @@ tuple make_tuple(Args &&...args_) { for (size_t i = 0; i < args.size(); i++) { if (!args[i]) { #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) - throw cast_error_unable_to_convert_call_arg(); + throw cast_error_unable_to_convert_call_arg(std::to_string(i)); #else std::array argtypes{{type_id()...}}; throw cast_error_unable_to_convert_call_arg(std::to_string(i), argtypes[i]); @@ -1508,7 +1547,7 @@ class unpacking_collector { detail::make_caster::cast(std::forward(x), policy, {})); if (!o) { #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) - throw cast_error_unable_to_convert_call_arg(); + throw cast_error_unable_to_convert_call_arg(std::to_string(args_list.size())); #else throw cast_error_unable_to_convert_call_arg(std::to_string(args_list.size()), type_id()); @@ -1540,12 +1579,12 @@ class unpacking_collector { } if (!a.value) { #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) - throw cast_error_unable_to_convert_call_arg(); + throw cast_error_unable_to_convert_call_arg(a.name); #else throw cast_error_unable_to_convert_call_arg(a.name, a.type); #endif } - m_kwargs[a.name] = a.value; + m_kwargs[a.name] = std::move(a.value); } void process(list & /*args_list*/, detail::kwargs_proxy kp) { diff --git a/pybind11/include/pybind11/detail/class.h b/pybind11/include/pybind11/detail/class.h index 42720f8..bc2b40c 100644 --- a/pybind11/include/pybind11/detail/class.h +++ b/pybind11/include/pybind11/detail/class.h @@ -55,6 +55,9 @@ extern "C" inline int pybind11_static_set(PyObject *self, PyObject *obj, PyObjec return PyProperty_Type.tp_descr_set(self, cls, value); } +// Forward declaration to use in `make_static_property_type()` +inline void enable_dynamic_attributes(PyHeapTypeObject *heap_type); + /** A `static_property` is the same as a `property` but the `__get__()` and `__set__()` methods are modified to always use the object type instead of a concrete instance. Return value: New reference. */ @@ -87,6 +90,13 @@ inline PyTypeObject *make_static_property_type() { pybind11_fail("make_static_property_type(): failure in PyType_Ready()!"); } +# if PY_VERSION_HEX >= 0x030C0000 + // PRE 3.12 FEATURE FREEZE. PLEASE REVIEW AFTER FREEZE. + // Since Python-3.12 property-derived types are required to + // have dynamic attributes (to set `__doc__`) + enable_dynamic_attributes(heap_type); +# endif + setattr((PyObject *) type, "__module__", str("pybind11_builtins")); PYBIND11_SET_OLDPY_QUALNAME(type, name_obj); @@ -435,9 +445,17 @@ inline void clear_instance(PyObject *self) { /// Instance destructor function for all pybind11 types. It calls `type_info.dealloc` /// to destroy the C++ object itself, while the rest is Python bookkeeping. extern "C" inline void pybind11_object_dealloc(PyObject *self) { + auto *type = Py_TYPE(self); + + // If this is a GC tracked object, untrack it first + // Note that the track call is implicitly done by the + // default tp_alloc, which we never override. + if (PyType_HasFeature(type, Py_TPFLAGS_HAVE_GC) != 0) { + PyObject_GC_UnTrack(self); + } + clear_instance(self); - auto *type = Py_TYPE(self); type->tp_free(self); #if PY_VERSION_HEX < 0x03080000 @@ -502,31 +520,6 @@ inline PyObject *make_object_base_type(PyTypeObject *metaclass) { return (PyObject *) heap_type; } -/// dynamic_attr: Support for `d = instance.__dict__`. -extern "C" inline PyObject *pybind11_get_dict(PyObject *self, void *) { - PyObject *&dict = *_PyObject_GetDictPtr(self); - if (!dict) { - dict = PyDict_New(); - } - Py_XINCREF(dict); - return dict; -} - -/// dynamic_attr: Support for `instance.__dict__ = dict()`. -extern "C" inline int pybind11_set_dict(PyObject *self, PyObject *new_dict, void *) { - if (!PyDict_Check(new_dict)) { - PyErr_Format(PyExc_TypeError, - "__dict__ must be set to a dictionary, not a '%.200s'", - get_fully_qualified_tp_name(Py_TYPE(new_dict)).c_str()); - return -1; - } - PyObject *&dict = *_PyObject_GetDictPtr(self); - Py_INCREF(new_dict); - Py_CLEAR(dict); - dict = new_dict; - return 0; -} - /// dynamic_attr: Allow the garbage collector to traverse the internal instance `__dict__`. extern "C" inline int pybind11_traverse(PyObject *self, visitproc visit, void *arg) { PyObject *&dict = *_PyObject_GetDictPtr(self); @@ -558,9 +551,17 @@ inline void enable_dynamic_attributes(PyHeapTypeObject *heap_type) { type->tp_traverse = pybind11_traverse; type->tp_clear = pybind11_clear; - static PyGetSetDef getset[] = { - {const_cast("__dict__"), pybind11_get_dict, pybind11_set_dict, nullptr, nullptr}, - {nullptr, nullptr, nullptr, nullptr, nullptr}}; + static PyGetSetDef getset[] = {{ +#if PY_VERSION_HEX < 0x03070000 + const_cast("__dict__"), +#else + "__dict__", +#endif + PyObject_GenericGetDict, + PyObject_GenericSetDict, + nullptr, + nullptr}, + {nullptr, nullptr, nullptr, nullptr, nullptr}}; type->tp_getset = getset; } diff --git a/pybind11/include/pybind11/detail/common.h b/pybind11/include/pybind11/detail/common.h index 1da323f..31a54c7 100644 --- a/pybind11/include/pybind11/detail/common.h +++ b/pybind11/include/pybind11/detail/common.h @@ -10,15 +10,76 @@ #pragma once #define PYBIND11_VERSION_MAJOR 2 -#define PYBIND11_VERSION_MINOR 10 -#define PYBIND11_VERSION_PATCH 0 +#define PYBIND11_VERSION_MINOR 11 +#define PYBIND11_VERSION_PATCH 1 // Similar to Python's convention: https://docs.python.org/3/c-api/apiabiversion.html // Additional convention: 0xD = dev -#define PYBIND11_VERSION_HEX 0x020A0000 +#define PYBIND11_VERSION_HEX 0x020B0100 + +// Define some generic pybind11 helper macros for warning management. +// +// Note that compiler-specific push/pop pairs are baked into the +// PYBIND11_NAMESPACE_BEGIN/PYBIND11_NAMESPACE_END pair of macros. Therefore manual +// PYBIND11_WARNING_PUSH/PYBIND11_WARNING_POP are usually only needed in `#include` sections. +// +// If you find you need to suppress a warning, please try to make the suppression as local as +// possible using these macros. Please also be sure to push/pop with the pybind11 macros. Please +// only use compiler specifics if you need to check specific versions, e.g. Apple Clang vs. vanilla +// Clang. +#if defined(_MSC_VER) +# define PYBIND11_COMPILER_MSVC +# define PYBIND11_PRAGMA(...) __pragma(__VA_ARGS__) +# define PYBIND11_WARNING_PUSH PYBIND11_PRAGMA(warning(push)) +# define PYBIND11_WARNING_POP PYBIND11_PRAGMA(warning(pop)) +#elif defined(__INTEL_COMPILER) +# define PYBIND11_COMPILER_INTEL +# define PYBIND11_PRAGMA(...) _Pragma(#__VA_ARGS__) +# define PYBIND11_WARNING_PUSH PYBIND11_PRAGMA(warning push) +# define PYBIND11_WARNING_POP PYBIND11_PRAGMA(warning pop) +#elif defined(__clang__) +# define PYBIND11_COMPILER_CLANG +# define PYBIND11_PRAGMA(...) _Pragma(#__VA_ARGS__) +# define PYBIND11_WARNING_PUSH PYBIND11_PRAGMA(clang diagnostic push) +# define PYBIND11_WARNING_POP PYBIND11_PRAGMA(clang diagnostic push) +#elif defined(__GNUC__) +# define PYBIND11_COMPILER_GCC +# define PYBIND11_PRAGMA(...) _Pragma(#__VA_ARGS__) +# define PYBIND11_WARNING_PUSH PYBIND11_PRAGMA(GCC diagnostic push) +# define PYBIND11_WARNING_POP PYBIND11_PRAGMA(GCC diagnostic pop) +#endif + +#ifdef PYBIND11_COMPILER_MSVC +# define PYBIND11_WARNING_DISABLE_MSVC(name) PYBIND11_PRAGMA(warning(disable : name)) +#else +# define PYBIND11_WARNING_DISABLE_MSVC(name) +#endif + +#ifdef PYBIND11_COMPILER_CLANG +# define PYBIND11_WARNING_DISABLE_CLANG(name) PYBIND11_PRAGMA(clang diagnostic ignored name) +#else +# define PYBIND11_WARNING_DISABLE_CLANG(name) +#endif + +#ifdef PYBIND11_COMPILER_GCC +# define PYBIND11_WARNING_DISABLE_GCC(name) PYBIND11_PRAGMA(GCC diagnostic ignored name) +#else +# define PYBIND11_WARNING_DISABLE_GCC(name) +#endif + +#ifdef PYBIND11_COMPILER_INTEL +# define PYBIND11_WARNING_DISABLE_INTEL(name) PYBIND11_PRAGMA(warning disable name) +#else +# define PYBIND11_WARNING_DISABLE_INTEL(name) +#endif -#define PYBIND11_NAMESPACE_BEGIN(name) namespace name { -#define PYBIND11_NAMESPACE_END(name) } +#define PYBIND11_NAMESPACE_BEGIN(name) \ + namespace name { \ + PYBIND11_WARNING_PUSH + +#define PYBIND11_NAMESPACE_END(name) \ + PYBIND11_WARNING_POP \ + } // Robust support for some features and loading modules compiled against different pybind versions // requires forcing hidden visibility on pybind code, so we enforce this by setting the attribute @@ -96,13 +157,10 @@ #endif #if !defined(PYBIND11_EXPORT_EXCEPTION) -# ifdef __MINGW32__ -// workaround for: -// error: 'dllexport' implies default visibility, but xxx has already been declared with a -// different visibility -# define PYBIND11_EXPORT_EXCEPTION -# else +# if defined(__apple_build_version__) # define PYBIND11_EXPORT_EXCEPTION PYBIND11_EXPORT +# else +# define PYBIND11_EXPORT_EXCEPTION # endif #endif @@ -154,9 +212,9 @@ /// Include Python header, disable linking to pythonX_d.lib on Windows in debug mode #if defined(_MSC_VER) -# pragma warning(push) +PYBIND11_WARNING_PUSH +PYBIND11_WARNING_DISABLE_MSVC(4505) // C4505: 'PySlice_GetIndicesEx': unreferenced local function has been removed (PyPy only) -# pragma warning(disable : 4505) # if defined(_DEBUG) && !defined(Py_DEBUG) // Workaround for a VS 2022 issue. // NOTE: This workaround knowingly violates the Python.h include order requirement: @@ -205,11 +263,8 @@ # endif #endif -#if defined(__cpp_lib_char8_t) && __cpp_lib_char8_t >= 201811L -# define PYBIND11_HAS_U8STRING -#endif - #include +// Reminder: WITH_THREAD is always defined if PY_VERSION_HEX >= 0x03070000 #if PY_VERSION_HEX < 0x03060000 # error "PYTHON < 3.6 IS UNSUPPORTED. pybind11 v2.9 was the last to support Python 2 and 3.5." #endif @@ -233,12 +288,16 @@ # undef copysign #endif +#if defined(PYPY_VERSION) && !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) +# define PYBIND11_SIMPLE_GIL_MANAGEMENT +#endif + #if defined(_MSC_VER) # if defined(PYBIND11_DEBUG_MARKER) # define _DEBUG # undef PYBIND11_DEBUG_MARKER # endif -# pragma warning(pop) +PYBIND11_WARNING_POP #endif #include @@ -259,6 +318,17 @@ # endif #endif +// Must be after including or one of the other headers specified by the standard +#if defined(__cpp_lib_char8_t) && __cpp_lib_char8_t >= 201811L +# define PYBIND11_HAS_U8STRING +#endif + +// See description of PR #4246: +#if !defined(PYBIND11_NO_ASSERT_GIL_HELD_INCREF_DECREF) && !defined(NDEBUG) \ + && !defined(PYPY_VERSION) && !defined(PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF) +# define PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF +#endif + // #define PYBIND11_STR_LEGACY_PERMISSIVE // If DEFINED, pybind11::str can hold PyUnicodeObject or PyBytesObject // (probably surprising and never documented, but this was the @@ -363,7 +433,7 @@ /** \rst This macro creates the entry point that will be invoked when the Python interpreter - imports an extension module. The module name is given as the fist argument and it + imports an extension module. The module name is given as the first argument and it should not be in quotes. The second macro argument defines a variable of type `py::module_` which can be used to initialize the module. @@ -588,6 +658,10 @@ template using remove_cvref_t = typename remove_cvref::type; #endif +/// Example usage: is_same_ignoring_cvref::value +template +using is_same_ignoring_cvref = std::is_same, U>; + /// Index sequences #if defined(PYBIND11_CPP14) using std::index_sequence; @@ -681,7 +755,16 @@ template struct remove_class { using type = R(A...); }; - +#ifdef __cpp_noexcept_function_type +template +struct remove_class { + using type = R(A...); +}; +template +struct remove_class { + using type = R(A...); +}; +#endif /// Helper template to strip away type modifiers template struct intrinsic_type { @@ -898,12 +981,6 @@ using expand_side_effects = bool[]; PYBIND11_NAMESPACE_END(detail) -#if defined(_MSC_VER) -# pragma warning(push) -# pragma warning(disable : 4275) -// warning C4275: An exported class was derived from a class that wasn't exported. -// Can be ignored when derived from a STL class. -#endif /// C++ bindings of builtin Python exceptions class PYBIND11_EXPORT_EXCEPTION builtin_exception : public std::runtime_error { public: @@ -911,9 +988,6 @@ class PYBIND11_EXPORT_EXCEPTION builtin_exception : public std::runtime_error { /// Set the error using the Python C API virtual void set_error() const = 0; }; -#if defined(_MSC_VER) -# pragma warning(pop) -#endif #define PYBIND11_RUNTIME_EXCEPTION(name, type) \ class PYBIND11_EXPORT_EXCEPTION name : public builtin_exception { \ @@ -948,6 +1022,15 @@ PYBIND11_RUNTIME_EXCEPTION(reference_cast_error, PyExc_RuntimeError) /// Used in template struct format_descriptor {}; +template +struct format_descriptor< + T, + detail::enable_if_t::value>> { + static constexpr const char c = 'O'; + static constexpr const char value[2] = {c, '\0'}; + static std::string format() { return std::string(1, c); } +}; + PYBIND11_NAMESPACE_BEGIN(detail) // Returns the index of the given type in the type char array below, and in the list in numpy.h // The order here is: bool; 8 ints ((signed,unsigned)x(8,16,32,64)bits); float,double,long double; @@ -1033,12 +1116,7 @@ PYBIND11_NAMESPACE_END(detail) /// - regular: static_cast(&Class::func) /// - sweet: overload_cast(&Class::func) template -# if (defined(_MSC_VER) && _MSC_VER < 1920) /* MSVC 2017 */ \ - || (defined(__clang__) && __clang_major__ == 5) -static constexpr detail::overload_cast_impl overload_cast = {}; -# else -static constexpr detail::overload_cast_impl overload_cast; -# endif +static constexpr detail::overload_cast_impl overload_cast{}; #endif /// Const member function selector for overload_cast @@ -1147,20 +1225,28 @@ constexpr # define PYBIND11_WORKAROUND_INCORRECT_GCC_UNUSED_BUT_SET_PARAMETER(...) #endif -#if defined(_MSC_VER) // All versions (as of July 2021). - -// warning C4127: Conditional expression is constant -constexpr inline bool silence_msvc_c4127(bool cond) { return cond; } - -# define PYBIND11_SILENCE_MSVC_C4127(...) ::pybind11::detail::silence_msvc_c4127(__VA_ARGS__) - -#else -# define PYBIND11_SILENCE_MSVC_C4127(...) __VA_ARGS__ +#if defined(__clang__) \ + && (defined(__apple_build_version__) /* AppleClang 13.0.0.13000029 was the only data point \ + available. */ \ + || (__clang_major__ >= 7 \ + && __clang_major__ <= 12) /* Clang 3, 5, 13, 14, 15 do not generate the warning. */ \ + ) +# define PYBIND11_DETECTED_CLANG_WITH_MISLEADING_CALL_STD_MOVE_EXPLICITLY_WARNING +// Example: +// tests/test_kwargs_and_defaults.cpp:46:68: error: local variable 'args' will be copied despite +// being returned by name [-Werror,-Wreturn-std-move] +// m.def("args_function", [](py::args args) -> py::tuple { return args; }); +// ^~~~ +// test_kwargs_and_defaults.cpp:46:68: note: call 'std::move' explicitly to avoid copying +// m.def("args_function", [](py::args args) -> py::tuple { return args; }); +// ^~~~ +// std::move(args) #endif // Pybind offers detailed error messages by default for all builts that are debug (through the -// negation of ndebug). This can also be manually enabled by users, for any builds, through -// defining PYBIND11_DETAILED_ERROR_MESSAGES. +// negation of NDEBUG). This can also be manually enabled by users, for any builds, through +// defining PYBIND11_DETAILED_ERROR_MESSAGES. This information is primarily useful for those +// who are writing (as opposed to merely using) libraries that use pybind11. #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) && !defined(NDEBUG) # define PYBIND11_DETAILED_ERROR_MESSAGES #endif diff --git a/pybind11/include/pybind11/detail/descr.h b/pybind11/include/pybind11/detail/descr.h index e7a5e2c..635614b 100644 --- a/pybind11/include/pybind11/detail/descr.h +++ b/pybind11/include/pybind11/detail/descr.h @@ -143,11 +143,24 @@ constexpr descr concat(const descr &descr) { return descr; } +#ifdef __cpp_fold_expressions +template +constexpr descr operator,(const descr &a, + const descr &b) { + return a + const_name(", ") + b; +} + +template +constexpr auto concat(const descr &d, const Args &...args) { + return (d, ..., args); +} +#else template constexpr auto concat(const descr &d, const Args &...args) -> decltype(std::declval>() + concat(args...)) { return d + const_name(", ") + concat(args...); } +#endif template constexpr descr type_descr(const descr &descr) { diff --git a/pybind11/include/pybind11/detail/init.h b/pybind11/include/pybind11/detail/init.h index 05f4fe5..e211716 100644 --- a/pybind11/include/pybind11/detail/init.h +++ b/pybind11/include/pybind11/detail/init.h @@ -12,6 +12,9 @@ #include "class.h" PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +PYBIND11_WARNING_DISABLE_MSVC(4127) + PYBIND11_NAMESPACE_BEGIN(detail) template <> @@ -115,7 +118,7 @@ template void construct(value_and_holder &v_h, Cpp *ptr, bool need_alias) { PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(need_alias); no_nullptr(ptr); - if (PYBIND11_SILENCE_MSVC_C4127(Class::has_alias) && need_alias && !is_alias(ptr)) { + if (Class::has_alias && need_alias && !is_alias(ptr)) { // We're going to try to construct an alias by moving the cpp type. Whether or not // that succeeds, we still need to destroy the original cpp pointer (either the // moved away leftover, if the alias construction works, or the value itself if we @@ -156,7 +159,7 @@ void construct(value_and_holder &v_h, Holder holder, bool need_alias) { auto *ptr = holder_helper>::get(holder); no_nullptr(ptr); // If we need an alias, check that the held pointer is actually an alias instance - if (PYBIND11_SILENCE_MSVC_C4127(Class::has_alias) && need_alias && !is_alias(ptr)) { + if (Class::has_alias && need_alias && !is_alias(ptr)) { throw type_error("pybind11::init(): construction failed: returned holder-wrapped instance " "is not an alias instance"); } @@ -172,9 +175,9 @@ void construct(value_and_holder &v_h, Holder holder, bool need_alias) { template void construct(value_and_holder &v_h, Cpp &&result, bool need_alias) { PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(need_alias); - static_assert(std::is_move_constructible>::value, + static_assert(is_move_constructible>::value, "pybind11::init() return-by-value factory function requires a movable class"); - if (PYBIND11_SILENCE_MSVC_C4127(Class::has_alias) && need_alias) { + if (Class::has_alias && need_alias) { construct_alias_from_cpp(is_alias_constructible{}, v_h, std::move(result)); } else { v_h.value_ptr() = new Cpp(std::move(result)); @@ -187,7 +190,7 @@ void construct(value_and_holder &v_h, Cpp &&result, bool need_alias) { template void construct(value_and_holder &v_h, Alias &&result, bool) { static_assert( - std::is_move_constructible>::value, + is_move_constructible>::value, "pybind11::init() return-by-alias-value factory function requires a movable alias class"); v_h.value_ptr() = new Alias(std::move(result)); } @@ -206,10 +209,11 @@ struct constructor { extra...); } - template , Args...>::value, - int> = 0> + template < + typename Class, + typename... Extra, + enable_if_t, Args...>::value, int> + = 0> static void execute(Class &cl, const Extra &...extra) { cl.def( "__init__", @@ -226,10 +230,11 @@ struct constructor { extra...); } - template , Args...>::value, - int> = 0> + template < + typename Class, + typename... Extra, + enable_if_t, Args...>::value, int> + = 0> static void execute(Class &cl, const Extra &...extra) { cl.def( "__init__", @@ -245,10 +250,11 @@ struct constructor { // Implementing class for py::init_alias<...>() template struct alias_constructor { - template , Args...>::value, - int> = 0> + template < + typename Class, + typename... Extra, + enable_if_t, Args...>::value, int> + = 0> static void execute(Class &cl, const Extra &...extra) { cl.def( "__init__", diff --git a/pybind11/include/pybind11/detail/internals.h b/pybind11/include/pybind11/detail/internals.h index 6ca5e14..aaa7f86 100644 --- a/pybind11/include/pybind11/detail/internals.h +++ b/pybind11/include/pybind11/detail/internals.h @@ -9,6 +9,12 @@ #pragma once +#include "common.h" + +#if defined(WITH_THREAD) && defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) +# include "../gil.h" +#endif + #include "../pytypes.h" #include @@ -28,15 +34,26 @@ /// further ABI-incompatible changes may be made before the ABI is officially /// changed to the new version. #ifndef PYBIND11_INTERNALS_VERSION -# define PYBIND11_INTERNALS_VERSION 4 +# if PY_VERSION_HEX >= 0x030C0000 +// Version bump for Python 3.12+, before first 3.12 beta release. +# define PYBIND11_INTERNALS_VERSION 5 +# else +# define PYBIND11_INTERNALS_VERSION 4 +# endif #endif +// This requirement is mainly to reduce the support burden (see PR #4570). +static_assert(PY_VERSION_HEX < 0x030C0000 || PYBIND11_INTERNALS_VERSION >= 5, + "pybind11 ABI version 5 is the minimum for Python 3.12+"); + PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) using ExceptionTranslator = void (*)(std::exception_ptr); PYBIND11_NAMESPACE_BEGIN(detail) +constexpr const char *internals_function_record_capsule_name = "pybind11_function_record_capsule"; + // Forward declarations inline PyTypeObject *make_static_property_type(); inline PyTypeObject *make_default_metaclass(); @@ -49,7 +66,7 @@ inline PyObject *make_object_base_type(PyTypeObject *metaclass); // `Py_LIMITED_API` anyway. # if PYBIND11_INTERNALS_VERSION > 4 # define PYBIND11_TLS_KEY_REF Py_tss_t & -# ifdef __GNUC__ +# if defined(__GNUC__) && !defined(__INTEL_COMPILER) // Clang on macOS warns due to `Py_tss_NEEDS_INIT` not specifying an initializer // for every field. # define PYBIND11_TLS_KEY_INIT(var) \ @@ -106,7 +123,8 @@ inline void tls_replace_value(PYBIND11_TLS_KEY_REF key, void *value) { // libstdc++, this doesn't happen: equality and the type_index hash are based on the type name, // which works. If not under a known-good stl, provide our own name-based hash and equality // functions that use the type name. -#if defined(__GLIBCXX__) +#if (PYBIND11_INTERNALS_VERSION <= 4 && defined(__GLIBCXX__)) \ + || (PYBIND11_INTERNALS_VERSION >= 5 && !defined(_LIBCPP_VERSION)) inline bool same_type(const std::type_info &lhs, const std::type_info &rhs) { return lhs == rhs; } using type_hash = std::hash; using type_equal_to = std::equal_to; @@ -169,11 +187,23 @@ struct internals { PyTypeObject *default_metaclass; PyObject *instance_base; #if defined(WITH_THREAD) + // Unused if PYBIND11_SIMPLE_GIL_MANAGEMENT is defined: PYBIND11_TLS_KEY_INIT(tstate) # if PYBIND11_INTERNALS_VERSION > 4 PYBIND11_TLS_KEY_INIT(loader_life_support_tls_key) # endif // PYBIND11_INTERNALS_VERSION > 4 + // Unused if PYBIND11_SIMPLE_GIL_MANAGEMENT is defined: PyInterpreterState *istate = nullptr; + +# if PYBIND11_INTERNALS_VERSION > 4 + // Note that we have to use a std::string to allocate memory to ensure a unique address + // We want unique addresses since we use pointer equality to compare function records + std::string function_record_capsule_name = internals_function_record_capsule_name; +# endif + + internals() = default; + internals(const internals &other) = delete; + internals &operator=(const internals &other) = delete; ~internals() { # if PYBIND11_INTERNALS_VERSION > 4 PYBIND11_TLS_FREE(loader_life_support_tls_key); @@ -401,6 +431,38 @@ inline void translate_local_exception(std::exception_ptr p) { } #endif +inline object get_python_state_dict() { + object state_dict; +#if PYBIND11_INTERNALS_VERSION <= 4 || PY_VERSION_HEX < 0x03080000 || defined(PYPY_VERSION) + state_dict = reinterpret_borrow(PyEval_GetBuiltins()); +#else +# if PY_VERSION_HEX < 0x03090000 + PyInterpreterState *istate = _PyInterpreterState_Get(); +# else + PyInterpreterState *istate = PyInterpreterState_Get(); +# endif + if (istate) { + state_dict = reinterpret_borrow(PyInterpreterState_GetDict(istate)); + } +#endif + if (!state_dict) { + raise_from(PyExc_SystemError, "pybind11::detail::get_python_state_dict() FAILED"); + } + return state_dict; +} + +inline object get_internals_obj_from_state_dict(handle state_dict) { + return reinterpret_borrow(dict_getitemstring(state_dict.ptr(), PYBIND11_INTERNALS_ID)); +} + +inline internals **get_internals_pp_from_capsule(handle obj) { + void *raw_ptr = PyCapsule_GetPointer(obj.ptr(), /*name=*/nullptr); + if (raw_ptr == nullptr) { + raise_from(PyExc_SystemError, "pybind11::detail::get_internals_pp_from_capsule() FAILED"); + } + return static_cast(raw_ptr); +} + /// Return a reference to the current `internals` data PYBIND11_NOINLINE internals &get_internals() { auto **&internals_pp = get_internals_pp(); @@ -408,21 +470,29 @@ PYBIND11_NOINLINE internals &get_internals() { return **internals_pp; } +#if defined(WITH_THREAD) +# if defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) + gil_scoped_acquire gil; +# else // Ensure that the GIL is held since we will need to make Python calls. // Cannot use py::gil_scoped_acquire here since that constructor calls get_internals. struct gil_scoped_acquire_local { gil_scoped_acquire_local() : state(PyGILState_Ensure()) {} + gil_scoped_acquire_local(const gil_scoped_acquire_local &) = delete; + gil_scoped_acquire_local &operator=(const gil_scoped_acquire_local &) = delete; ~gil_scoped_acquire_local() { PyGILState_Release(state); } const PyGILState_STATE state; } gil; +# endif +#endif error_scope err_scope; - PYBIND11_STR_TYPE id(PYBIND11_INTERNALS_ID); - auto builtins = handle(PyEval_GetBuiltins()); - if (builtins.contains(id) && isinstance(builtins[id])) { - internals_pp = static_cast(capsule(builtins[id])); - - // We loaded builtins through python's builtins, which means that our `error_already_set` + dict state_dict = get_python_state_dict(); + if (object internals_obj = get_internals_obj_from_state_dict(state_dict)) { + internals_pp = get_internals_pp_from_capsule(internals_obj); + } + if (internals_pp && *internals_pp) { + // We loaded the internals through `state_dict`, which means that our `error_already_set` // and `builtin_exception` may be different local classes than the ones set up in the // initial exception translator, below, so add another for our local exception classes. // @@ -440,16 +510,15 @@ PYBIND11_NOINLINE internals &get_internals() { internals_ptr = new internals(); #if defined(WITH_THREAD) -# if PY_VERSION_HEX < 0x03090000 - PyEval_InitThreads(); -# endif PyThreadState *tstate = PyThreadState_Get(); + // NOLINTNEXTLINE(bugprone-assignment-in-if-condition) if (!PYBIND11_TLS_KEY_CREATE(internals_ptr->tstate)) { pybind11_fail("get_internals: could not successfully initialize the tstate TSS key!"); } PYBIND11_TLS_REPLACE_VALUE(internals_ptr->tstate, tstate); # if PYBIND11_INTERNALS_VERSION > 4 + // NOLINTNEXTLINE(bugprone-assignment-in-if-condition) if (!PYBIND11_TLS_KEY_CREATE(internals_ptr->loader_life_support_tls_key)) { pybind11_fail("get_internals: could not successfully initialize the " "loader_life_support TSS key!"); @@ -457,7 +526,7 @@ PYBIND11_NOINLINE internals &get_internals() { # endif internals_ptr->istate = tstate->interp; #endif - builtins[id] = capsule(internals_pp); + state_dict[PYBIND11_INTERNALS_ID] = capsule(internals_pp); internals_ptr->registered_exception_translators.push_front(&translate_exception); internals_ptr->static_property_type = make_static_property_type(); internals_ptr->default_metaclass = make_default_metaclass(); @@ -489,6 +558,7 @@ struct local_internals { struct shared_loader_life_support_data { PYBIND11_TLS_KEY_INIT(loader_life_support_tls_key) shared_loader_life_support_data() { + // NOLINTNEXTLINE(bugprone-assignment-in-if-condition) if (!PYBIND11_TLS_KEY_CREATE(loader_life_support_tls_key)) { pybind11_fail("local_internals: could not successfully initialize the " "loader_life_support TLS key!"); @@ -512,8 +582,13 @@ struct local_internals { /// Works like `get_internals`, but for things which are locally registered. inline local_internals &get_local_internals() { - static local_internals locals; - return locals; + // Current static can be created in the interpreter finalization routine. If the later will be + // destroyed in another static variable destructor, creation of this static there will cause + // static deinitialization fiasco. In order to avoid it we avoid destruction of the + // local_internals static. One can read more about the problem and current solution here: + // https://google.github.io/styleguide/cppguide.html#Static_and_Global_Variables + static auto *locals = new local_internals(); + return *locals; } /// Constructs a std::string with the given arguments, stores it in `internals`, and returns its @@ -527,6 +602,25 @@ const char *c_str(Args &&...args) { return strings.front().c_str(); } +inline const char *get_function_record_capsule_name() { +#if PYBIND11_INTERNALS_VERSION > 4 + return get_internals().function_record_capsule_name.c_str(); +#else + return nullptr; +#endif +} + +// Determine whether or not the following capsule contains a pybind11 function record. +// Note that we use `internals` to make sure that only ABI compatible records are touched. +// +// This check is currently used in two places: +// - An important optimization in functional.h to avoid overhead in C++ -> Python -> C++ +// - The sibling feature of cpp_function to allow overloads +inline bool is_function_record_capsule(const capsule &cap) { + // Pointer equality as we rely on internals() to ensure unique pointers + return cap.name() == get_function_record_capsule_name(); +} + PYBIND11_NAMESPACE_END(detail) /// Returns a named pointer that is shared among all extension modules (using the same diff --git a/pybind11/include/pybind11/detail/type_caster_base.h b/pybind11/include/pybind11/detail/type_caster_base.h index 21f69c2..1638750 100644 --- a/pybind11/include/pybind11/detail/type_caster_base.h +++ b/pybind11/include/pybind11/detail/type_caster_base.h @@ -258,9 +258,9 @@ struct value_and_holder { // Main constructor for a found value/holder: value_and_holder(instance *i, const detail::type_info *type, size_t vpos, size_t index) - : inst{i}, index{index}, type{type}, vh{inst->simple_layout - ? inst->simple_value_holder - : &inst->nonsimple.values_and_holders[vpos]} {} + : inst{i}, index{index}, type{type}, + vh{inst->simple_layout ? inst->simple_value_holder + : &inst->nonsimple.values_and_holders[vpos]} {} // Default constructor (used to signal a value-and-holder not found by get_value_and_holder()) value_and_holder() = default; @@ -822,23 +822,179 @@ using movable_cast_op_type typename std::add_rvalue_reference>::type, typename std::add_lvalue_reference>::type>>; -// std::is_copy_constructible isn't quite enough: it lets std::vector (and similar) through when -// T is non-copyable, but code containing such a copy constructor fails to actually compile. -template -struct is_copy_constructible : std::is_copy_constructible {}; +// Does the container have a mapped type and is it recursive? +// Implemented by specializations below. +template +struct container_mapped_type_traits { + static constexpr bool has_mapped_type = false; + static constexpr bool has_recursive_mapped_type = false; +}; + +template +struct container_mapped_type_traits< + Container, + typename std::enable_if< + std::is_same::value>::type> { + static constexpr bool has_mapped_type = true; + static constexpr bool has_recursive_mapped_type = true; +}; + +template +struct container_mapped_type_traits< + Container, + typename std::enable_if< + negation>::value>::type> { + static constexpr bool has_mapped_type = true; + static constexpr bool has_recursive_mapped_type = false; +}; + +// Does the container have a value type and is it recursive? +// Implemented by specializations below. +template +struct container_value_type_traits : std::false_type { + static constexpr bool has_value_type = false; + static constexpr bool has_recursive_value_type = false; +}; + +template +struct container_value_type_traits< + Container, + typename std::enable_if< + std::is_same::value>::type> { + static constexpr bool has_value_type = true; + static constexpr bool has_recursive_value_type = true; +}; + +template +struct container_value_type_traits< + Container, + typename std::enable_if< + negation>::value>::type> { + static constexpr bool has_value_type = true; + static constexpr bool has_recursive_value_type = false; +}; + +/* + * Tag to be used for representing the bottom of recursively defined types. + * Define this tag so we don't have to use void. + */ +struct recursive_bottom {}; + +/* + * Implementation detail of `recursive_container_traits` below. + * `T` is the `value_type` of the container, which might need to be modified to + * avoid recursive types and const types. + */ +template +struct impl_type_to_check_recursively { + /* + * If the container is recursive, then no further recursion should be done. + */ + using if_recursive = recursive_bottom; + /* + * Otherwise yield `T` unchanged. + */ + using if_not_recursive = T; +}; + +/* + * For pairs - only as value type of a map -, the first type should remove the `const`. + * Also, if the map is recursive, then the recursive checking should consider + * the first type only. + */ +template +struct impl_type_to_check_recursively, /* is_this_a_map = */ true> { + using if_recursive = typename std::remove_const::type; + using if_not_recursive = std::pair::type, B>; +}; + +/* + * Implementation of `recursive_container_traits` below. + */ +template +struct impl_recursive_container_traits { + using type_to_check_recursively = recursive_bottom; +}; -// Specialization for types that appear to be copy constructible but also look like stl containers -// (we specifically check for: has `value_type` and `reference` with `reference = value_type&`): if -// so, copy constructability depends on whether the value_type is copy constructible. template -struct is_copy_constructible< +struct impl_recursive_container_traits< Container, - enable_if_t< - all_of, - std::is_same, - // Avoid infinite recursion - negation>>::value>> - : is_copy_constructible {}; + typename std::enable_if::has_value_type>::type> { + static constexpr bool is_recursive + = container_mapped_type_traits::has_recursive_mapped_type + || container_value_type_traits::has_recursive_value_type; + /* + * This member dictates which type Pybind11 should check recursively in traits + * such as `is_move_constructible`, `is_copy_constructible`, `is_move_assignable`, ... + * Direct access to `value_type` should be avoided: + * 1. `value_type` might recursively contain the type again + * 2. `value_type` of STL map types is `std::pair`, the `const` + * should be removed. + * + */ + using type_to_check_recursively = typename std::conditional< + is_recursive, + typename impl_type_to_check_recursively< + typename Container::value_type, + container_mapped_type_traits::has_mapped_type>::if_recursive, + typename impl_type_to_check_recursively< + typename Container::value_type, + container_mapped_type_traits::has_mapped_type>::if_not_recursive>::type; +}; + +/* + * This trait defines the `type_to_check_recursively` which is needed to properly + * handle recursively defined traits such as `is_move_constructible` without going + * into an infinite recursion. + * Should be used instead of directly accessing the `value_type`. + * It cancels the recursion by returning the `recursive_bottom` tag. + * + * The default definition of `type_to_check_recursively` is as follows: + * + * 1. By default, it is `recursive_bottom`, so that the recursion is canceled. + * 2. If the type is non-recursive and defines a `value_type`, then the `value_type` is used. + * If the `value_type` is a pair and a `mapped_type` is defined, + * then the `const` is removed from the first type. + * 3. If the type is recursive and `value_type` is not a pair, then `recursive_bottom` is returned. + * 4. If the type is recursive and `value_type` is a pair and a `mapped_type` is defined, + * then `const` is removed from the first type and the first type is returned. + * + * This behavior can be extended by the user as seen in test_stl_binders.cpp. + * + * This struct is exactly the same as impl_recursive_container_traits. + * The duplication achieves that user-defined specializations don't compete + * with internal specializations, but take precedence. + */ +template +struct recursive_container_traits : impl_recursive_container_traits {}; + +template +struct is_move_constructible + : all_of, + is_move_constructible< + typename recursive_container_traits::type_to_check_recursively>> {}; + +template <> +struct is_move_constructible : std::true_type {}; + +// Likewise for std::pair +// (after C++17 it is mandatory that the move constructor not exist when the two types aren't +// themselves move constructible, but this can not be relied upon when T1 or T2 are themselves +// containers). +template +struct is_move_constructible> + : all_of, is_move_constructible> {}; + +// std::is_copy_constructible isn't quite enough: it lets std::vector (and similar) through when +// T is non-copyable, but code containing such a copy constructor fails to actually compile. +template +struct is_copy_constructible + : all_of, + is_copy_constructible< + typename recursive_container_traits::type_to_check_recursively>> {}; + +template <> +struct is_copy_constructible : std::true_type {}; // Likewise for std::pair // (after C++17 it is mandatory that the copy constructor not exist when the two types aren't @@ -849,14 +1005,16 @@ struct is_copy_constructible> : all_of, is_copy_constructible> {}; // The same problems arise with std::is_copy_assignable, so we use the same workaround. -template -struct is_copy_assignable : std::is_copy_assignable {}; -template -struct is_copy_assignable, - std::is_same>::value>> - : is_copy_assignable {}; +template +struct is_copy_assignable + : all_of< + std::is_copy_assignable, + is_copy_assignable::type_to_check_recursively>> { +}; + +template <> +struct is_copy_assignable : std::true_type {}; + template struct is_copy_assignable> : all_of, is_copy_assignable> {}; @@ -994,7 +1152,7 @@ class type_caster_base : public type_caster_generic { return [](const void *arg) -> void * { return new T(*reinterpret_cast(arg)); }; } - template ::value>> + template ::value>> static auto make_move_constructor(const T *) -> decltype(new T(std::declval()), Constructor{}) { return [](const void *arg) -> void * { @@ -1006,5 +1164,14 @@ class type_caster_base : public type_caster_generic { static Constructor make_move_constructor(...) { return nullptr; } }; +PYBIND11_NOINLINE std::string type_info_description(const std::type_info &ti) { + if (auto *type_data = get_type_info(ti)) { + handle th((PyObject *) type_data->type); + return th.attr("__module__").cast() + '.' + + th.attr("__qualname__").cast(); + } + return clean_type_id(ti.name()); +} + PYBIND11_NAMESPACE_END(detail) PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/pybind11/include/pybind11/eigen.h b/pybind11/include/pybind11/eigen.h index 4d221b3..273b9c9 100644 --- a/pybind11/include/pybind11/eigen.h +++ b/pybind11/include/pybind11/eigen.h @@ -9,700 +9,4 @@ #pragma once -/* HINT: To suppress warnings originating from the Eigen headers, use -isystem. - See also: - https://stackoverflow.com/questions/2579576/i-dir-vs-isystem-dir - https://stackoverflow.com/questions/1741816/isystem-for-ms-visual-studio-c-compiler -*/ - -#include "numpy.h" - -// The C4127 suppression was introduced for Eigen 3.4.0. In theory we could -// make it version specific, or even remove it later, but considering that -// 1. C4127 is generally far more distracting than useful for modern template code, and -// 2. we definitely want to ignore any MSVC warnings originating from Eigen code, -// it is probably best to keep this around indefinitely. -#if defined(_MSC_VER) -# pragma warning(push) -# pragma warning(disable : 4127) // C4127: conditional expression is constant -# pragma warning(disable : 5054) // https://github.com/pybind/pybind11/pull/3741 -// C5054: operator '&': deprecated between enumerations of different types -#endif - -#include -#include - -#if defined(_MSC_VER) -# pragma warning(pop) -#endif - -// Eigen prior to 3.2.7 doesn't have proper move constructors--but worse, some classes get implicit -// move constructors that break things. We could detect this an explicitly copy, but an extra copy -// of matrices seems highly undesirable. -static_assert(EIGEN_VERSION_AT_LEAST(3, 2, 7), - "Eigen support in pybind11 requires Eigen >= 3.2.7"); - -PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) - -// Provide a convenience alias for easier pass-by-ref usage with fully dynamic strides: -using EigenDStride = Eigen::Stride; -template -using EigenDRef = Eigen::Ref; -template -using EigenDMap = Eigen::Map; - -PYBIND11_NAMESPACE_BEGIN(detail) - -#if EIGEN_VERSION_AT_LEAST(3, 3, 0) -using EigenIndex = Eigen::Index; -template -using EigenMapSparseMatrix = Eigen::Map>; -#else -using EigenIndex = EIGEN_DEFAULT_DENSE_INDEX_TYPE; -template -using EigenMapSparseMatrix = Eigen::MappedSparseMatrix; -#endif - -// Matches Eigen::Map, Eigen::Ref, blocks, etc: -template -using is_eigen_dense_map = all_of, - std::is_base_of, T>>; -template -using is_eigen_mutable_map = std::is_base_of, T>; -template -using is_eigen_dense_plain - = all_of>, is_template_base_of>; -template -using is_eigen_sparse = is_template_base_of; -// Test for objects inheriting from EigenBase that aren't captured by the above. This -// basically covers anything that can be assigned to a dense matrix but that don't have a typical -// matrix data layout that can be copied from their .data(). For example, DiagonalMatrix and -// SelfAdjointView fall into this category. -template -using is_eigen_other - = all_of, - negation, is_eigen_dense_plain, is_eigen_sparse>>>; - -// Captures numpy/eigen conformability status (returned by EigenProps::conformable()): -template -struct EigenConformable { - bool conformable = false; - EigenIndex rows = 0, cols = 0; - EigenDStride stride{0, 0}; // Only valid if negativestrides is false! - bool negativestrides = false; // If true, do not use stride! - - // NOLINTNEXTLINE(google-explicit-constructor) - EigenConformable(bool fits = false) : conformable{fits} {} - // Matrix type: - EigenConformable(EigenIndex r, EigenIndex c, EigenIndex rstride, EigenIndex cstride) - : conformable{true}, rows{r}, cols{c}, - // TODO: when Eigen bug #747 is fixed, remove the tests for non-negativity. - // http://eigen.tuxfamily.org/bz/show_bug.cgi?id=747 - stride{EigenRowMajor ? (rstride > 0 ? rstride : 0) - : (cstride > 0 ? cstride : 0) /* outer stride */, - EigenRowMajor ? (cstride > 0 ? cstride : 0) - : (rstride > 0 ? rstride : 0) /* inner stride */}, - negativestrides{rstride < 0 || cstride < 0} {} - // Vector type: - EigenConformable(EigenIndex r, EigenIndex c, EigenIndex stride) - : EigenConformable(r, c, r == 1 ? c * stride : stride, c == 1 ? r : r * stride) {} - - template - bool stride_compatible() const { - // To have compatible strides, we need (on both dimensions) one of fully dynamic strides, - // matching strides, or a dimension size of 1 (in which case the stride value is - // irrelevant). Alternatively, if any dimension size is 0, the strides are not relevant - // (and numpy β‰₯ 1.23 sets the strides to 0 in that case, so we need to check explicitly). - if (negativestrides) { - return false; - } - if (rows == 0 || cols == 0) { - return true; - } - return (props::inner_stride == Eigen::Dynamic || props::inner_stride == stride.inner() - || (EigenRowMajor ? cols : rows) == 1) - && (props::outer_stride == Eigen::Dynamic || props::outer_stride == stride.outer() - || (EigenRowMajor ? rows : cols) == 1); - } - // NOLINTNEXTLINE(google-explicit-constructor) - operator bool() const { return conformable; } -}; - -template -struct eigen_extract_stride { - using type = Type; -}; -template -struct eigen_extract_stride> { - using type = StrideType; -}; -template -struct eigen_extract_stride> { - using type = StrideType; -}; - -// Helper struct for extracting information from an Eigen type -template -struct EigenProps { - using Type = Type_; - using Scalar = typename Type::Scalar; - using StrideType = typename eigen_extract_stride::type; - static constexpr EigenIndex rows = Type::RowsAtCompileTime, cols = Type::ColsAtCompileTime, - size = Type::SizeAtCompileTime; - static constexpr bool row_major = Type::IsRowMajor, - vector - = Type::IsVectorAtCompileTime, // At least one dimension has fixed size 1 - fixed_rows = rows != Eigen::Dynamic, fixed_cols = cols != Eigen::Dynamic, - fixed = size != Eigen::Dynamic, // Fully-fixed size - dynamic = !fixed_rows && !fixed_cols; // Fully-dynamic size - - template - using if_zero = std::integral_constant; - static constexpr EigenIndex inner_stride - = if_zero::value, - outer_stride = if_zero < StrideType::OuterStrideAtCompileTime, - vector ? size - : row_major ? cols - : rows > ::value; - static constexpr bool dynamic_stride - = inner_stride == Eigen::Dynamic && outer_stride == Eigen::Dynamic; - static constexpr bool requires_row_major - = !dynamic_stride && !vector && (row_major ? inner_stride : outer_stride) == 1; - static constexpr bool requires_col_major - = !dynamic_stride && !vector && (row_major ? outer_stride : inner_stride) == 1; - - // Takes an input array and determines whether we can make it fit into the Eigen type. If - // the array is a vector, we attempt to fit it into either an Eigen 1xN or Nx1 vector - // (preferring the latter if it will fit in either, i.e. for a fully dynamic matrix type). - static EigenConformable conformable(const array &a) { - const auto dims = a.ndim(); - if (dims < 1 || dims > 2) { - return false; - } - - if (dims == 2) { // Matrix type: require exact match (or dynamic) - - EigenIndex np_rows = a.shape(0), np_cols = a.shape(1), - np_rstride = a.strides(0) / static_cast(sizeof(Scalar)), - np_cstride = a.strides(1) / static_cast(sizeof(Scalar)); - if ((PYBIND11_SILENCE_MSVC_C4127(fixed_rows) && np_rows != rows) - || (PYBIND11_SILENCE_MSVC_C4127(fixed_cols) && np_cols != cols)) { - return false; - } - - return {np_rows, np_cols, np_rstride, np_cstride}; - } - - // Otherwise we're storing an n-vector. Only one of the strides will be used, but - // whichever is used, we want the (single) numpy stride value. - const EigenIndex n = a.shape(0), - stride = a.strides(0) / static_cast(sizeof(Scalar)); - - if (vector) { // Eigen type is a compile-time vector - if (PYBIND11_SILENCE_MSVC_C4127(fixed) && size != n) { - return false; // Vector size mismatch - } - return {rows == 1 ? 1 : n, cols == 1 ? 1 : n, stride}; - } - if (fixed) { - // The type has a fixed size, but is not a vector: abort - return false; - } - if (fixed_cols) { - // Since this isn't a vector, cols must be != 1. We allow this only if it exactly - // equals the number of elements (rows is Dynamic, and so 1 row is allowed). - if (cols != n) { - return false; - } - return {1, n, stride}; - } // Otherwise it's either fully dynamic, or column dynamic; both become a column vector - if (PYBIND11_SILENCE_MSVC_C4127(fixed_rows) && rows != n) { - return false; - } - return {n, 1, stride}; - } - - static constexpr bool show_writeable - = is_eigen_dense_map::value && is_eigen_mutable_map::value; - static constexpr bool show_order = is_eigen_dense_map::value; - static constexpr bool show_c_contiguous = show_order && requires_row_major; - static constexpr bool show_f_contiguous - = !show_c_contiguous && show_order && requires_col_major; - - static constexpr auto descriptor - = const_name("numpy.ndarray[") + npy_format_descriptor::name + const_name("[") - + const_name(const_name<(size_t) rows>(), const_name("m")) + const_name(", ") - + const_name(const_name<(size_t) cols>(), const_name("n")) + const_name("]") - + - // For a reference type (e.g. Ref) we have other constraints that might need to - // be satisfied: writeable=True (for a mutable reference), and, depending on the map's - // stride options, possibly f_contiguous or c_contiguous. We include them in the - // descriptor output to provide some hint as to why a TypeError is occurring (otherwise - // it can be confusing to see that a function accepts a 'numpy.ndarray[float64[3,2]]' and - // an error message that you *gave* a numpy.ndarray of the right type and dimensions. - const_name(", flags.writeable", "") - + const_name(", flags.c_contiguous", "") - + const_name(", flags.f_contiguous", "") + const_name("]"); -}; - -// Casts an Eigen type to numpy array. If given a base, the numpy array references the src data, -// otherwise it'll make a copy. writeable lets you turn off the writeable flag for the array. -template -handle -eigen_array_cast(typename props::Type const &src, handle base = handle(), bool writeable = true) { - constexpr ssize_t elem_size = sizeof(typename props::Scalar); - array a; - if (props::vector) { - a = array({src.size()}, {elem_size * src.innerStride()}, src.data(), base); - } else { - a = array({src.rows(), src.cols()}, - {elem_size * src.rowStride(), elem_size * src.colStride()}, - src.data(), - base); - } - - if (!writeable) { - array_proxy(a.ptr())->flags &= ~detail::npy_api::NPY_ARRAY_WRITEABLE_; - } - - return a.release(); -} - -// Takes an lvalue ref to some Eigen type and a (python) base object, creating a numpy array that -// reference the Eigen object's data with `base` as the python-registered base class (if omitted, -// the base will be set to None, and lifetime management is up to the caller). The numpy array is -// non-writeable if the given type is const. -template -handle eigen_ref_array(Type &src, handle parent = none()) { - // none here is to get past array's should-we-copy detection, which currently always - // copies when there is no base. Setting the base to None should be harmless. - return eigen_array_cast(src, parent, !std::is_const::value); -} - -// Takes a pointer to some dense, plain Eigen type, builds a capsule around it, then returns a -// numpy array that references the encapsulated data with a python-side reference to the capsule to -// tie its destruction to that of any dependent python objects. Const-ness is determined by -// whether or not the Type of the pointer given is const. -template ::value>> -handle eigen_encapsulate(Type *src) { - capsule base(src, [](void *o) { delete static_cast(o); }); - return eigen_ref_array(*src, base); -} - -// Type caster for regular, dense matrix types (e.g. MatrixXd), but not maps/refs/etc. of dense -// types. -template -struct type_caster::value>> { - using Scalar = typename Type::Scalar; - using props = EigenProps; - - bool load(handle src, bool convert) { - // If we're in no-convert mode, only load if given an array of the correct type - if (!convert && !isinstance>(src)) { - return false; - } - - // Coerce into an array, but don't do type conversion yet; the copy below handles it. - auto buf = array::ensure(src); - - if (!buf) { - return false; - } - - auto dims = buf.ndim(); - if (dims < 1 || dims > 2) { - return false; - } - - auto fits = props::conformable(buf); - if (!fits) { - return false; - } - - // Allocate the new type, then build a numpy reference into it - value = Type(fits.rows, fits.cols); - auto ref = reinterpret_steal(eigen_ref_array(value)); - if (dims == 1) { - ref = ref.squeeze(); - } else if (ref.ndim() == 1) { - buf = buf.squeeze(); - } - - int result = detail::npy_api::get().PyArray_CopyInto_(ref.ptr(), buf.ptr()); - - if (result < 0) { // Copy failed! - PyErr_Clear(); - return false; - } - - return true; - } - -private: - // Cast implementation - template - static handle cast_impl(CType *src, return_value_policy policy, handle parent) { - switch (policy) { - case return_value_policy::take_ownership: - case return_value_policy::automatic: - return eigen_encapsulate(src); - case return_value_policy::move: - return eigen_encapsulate(new CType(std::move(*src))); - case return_value_policy::copy: - return eigen_array_cast(*src); - case return_value_policy::reference: - case return_value_policy::automatic_reference: - return eigen_ref_array(*src); - case return_value_policy::reference_internal: - return eigen_ref_array(*src, parent); - default: - throw cast_error("unhandled return_value_policy: should not happen!"); - }; - } - -public: - // Normal returned non-reference, non-const value: - static handle cast(Type &&src, return_value_policy /* policy */, handle parent) { - return cast_impl(&src, return_value_policy::move, parent); - } - // If you return a non-reference const, we mark the numpy array readonly: - static handle cast(const Type &&src, return_value_policy /* policy */, handle parent) { - return cast_impl(&src, return_value_policy::move, parent); - } - // lvalue reference return; default (automatic) becomes copy - static handle cast(Type &src, return_value_policy policy, handle parent) { - if (policy == return_value_policy::automatic - || policy == return_value_policy::automatic_reference) { - policy = return_value_policy::copy; - } - return cast_impl(&src, policy, parent); - } - // const lvalue reference return; default (automatic) becomes copy - static handle cast(const Type &src, return_value_policy policy, handle parent) { - if (policy == return_value_policy::automatic - || policy == return_value_policy::automatic_reference) { - policy = return_value_policy::copy; - } - return cast(&src, policy, parent); - } - // non-const pointer return - static handle cast(Type *src, return_value_policy policy, handle parent) { - return cast_impl(src, policy, parent); - } - // const pointer return - static handle cast(const Type *src, return_value_policy policy, handle parent) { - return cast_impl(src, policy, parent); - } - - static constexpr auto name = props::descriptor; - - // NOLINTNEXTLINE(google-explicit-constructor) - operator Type *() { return &value; } - // NOLINTNEXTLINE(google-explicit-constructor) - operator Type &() { return value; } - // NOLINTNEXTLINE(google-explicit-constructor) - operator Type &&() && { return std::move(value); } - template - using cast_op_type = movable_cast_op_type; - -private: - Type value; -}; - -// Base class for casting reference/map/block/etc. objects back to python. -template -struct eigen_map_caster { -private: - using props = EigenProps; - -public: - // Directly referencing a ref/map's data is a bit dangerous (whatever the map/ref points to has - // to stay around), but we'll allow it under the assumption that you know what you're doing - // (and have an appropriate keep_alive in place). We return a numpy array pointing directly at - // the ref's data (The numpy array ends up read-only if the ref was to a const matrix type.) - // Note that this means you need to ensure you don't destroy the object in some other way (e.g. - // with an appropriate keep_alive, or with a reference to a statically allocated matrix). - static handle cast(const MapType &src, return_value_policy policy, handle parent) { - switch (policy) { - case return_value_policy::copy: - return eigen_array_cast(src); - case return_value_policy::reference_internal: - return eigen_array_cast(src, parent, is_eigen_mutable_map::value); - case return_value_policy::reference: - case return_value_policy::automatic: - case return_value_policy::automatic_reference: - return eigen_array_cast(src, none(), is_eigen_mutable_map::value); - default: - // move, take_ownership don't make any sense for a ref/map: - pybind11_fail("Invalid return_value_policy for Eigen Map/Ref/Block type"); - } - } - - static constexpr auto name = props::descriptor; - - // Explicitly delete these: support python -> C++ conversion on these (i.e. these can be return - // types but not bound arguments). We still provide them (with an explicitly delete) so that - // you end up here if you try anyway. - bool load(handle, bool) = delete; - operator MapType() = delete; - template - using cast_op_type = MapType; -}; - -// We can return any map-like object (but can only load Refs, specialized next): -template -struct type_caster::value>> : eigen_map_caster {}; - -// Loader for Ref<...> arguments. See the documentation for info on how to make this work without -// copying (it requires some extra effort in many cases). -template -struct type_caster< - Eigen::Ref, - enable_if_t>::value>> - : public eigen_map_caster> { -private: - using Type = Eigen::Ref; - using props = EigenProps; - using Scalar = typename props::Scalar; - using MapType = Eigen::Map; - using Array - = array_t; - static constexpr bool need_writeable = is_eigen_mutable_map::value; - // Delay construction (these have no default constructor) - std::unique_ptr map; - std::unique_ptr ref; - // Our array. When possible, this is just a numpy array pointing to the source data, but - // sometimes we can't avoid copying (e.g. input is not a numpy array at all, has an - // incompatible layout, or is an array of a type that needs to be converted). Using a numpy - // temporary (rather than an Eigen temporary) saves an extra copy when we need both type - // conversion and storage order conversion. (Note that we refuse to use this temporary copy - // when loading an argument for a Ref with M non-const, i.e. a read-write reference). - Array copy_or_ref; - -public: - bool load(handle src, bool convert) { - // First check whether what we have is already an array of the right type. If not, we - // can't avoid a copy (because the copy is also going to do type conversion). - bool need_copy = !isinstance(src); - - EigenConformable fits; - if (!need_copy) { - // We don't need a converting copy, but we also need to check whether the strides are - // compatible with the Ref's stride requirements - auto aref = reinterpret_borrow(src); - - if (aref && (!need_writeable || aref.writeable())) { - fits = props::conformable(aref); - if (!fits) { - return false; // Incompatible dimensions - } - if (!fits.template stride_compatible()) { - need_copy = true; - } else { - copy_or_ref = std::move(aref); - } - } else { - need_copy = true; - } - } - - if (need_copy) { - // We need to copy: If we need a mutable reference, or we're not supposed to convert - // (either because we're in the no-convert overload pass, or because we're explicitly - // instructed not to copy (via `py::arg().noconvert()`) we have to fail loading. - if (!convert || need_writeable) { - return false; - } - - Array copy = Array::ensure(src); - if (!copy) { - return false; - } - fits = props::conformable(copy); - if (!fits || !fits.template stride_compatible()) { - return false; - } - copy_or_ref = std::move(copy); - loader_life_support::add_patient(copy_or_ref); - } - - ref.reset(); - map.reset(new MapType(data(copy_or_ref), - fits.rows, - fits.cols, - make_stride(fits.stride.outer(), fits.stride.inner()))); - ref.reset(new Type(*map)); - - return true; - } - - // NOLINTNEXTLINE(google-explicit-constructor) - operator Type *() { return ref.get(); } - // NOLINTNEXTLINE(google-explicit-constructor) - operator Type &() { return *ref; } - template - using cast_op_type = pybind11::detail::cast_op_type<_T>; - -private: - template ::value, int> = 0> - Scalar *data(Array &a) { - return a.mutable_data(); - } - - template ::value, int> = 0> - const Scalar *data(Array &a) { - return a.data(); - } - - // Attempt to figure out a constructor of `Stride` that will work. - // If both strides are fixed, use a default constructor: - template - using stride_ctor_default = bool_constant::value>; - // Otherwise, if there is a two-index constructor, assume it is (outer,inner) like - // Eigen::Stride, and use it: - template - using stride_ctor_dual - = bool_constant::value - && std::is_constructible::value>; - // Otherwise, if there is a one-index constructor, and just one of the strides is dynamic, use - // it (passing whichever stride is dynamic). - template - using stride_ctor_outer - = bool_constant, stride_ctor_dual>::value - && S::OuterStrideAtCompileTime == Eigen::Dynamic - && S::InnerStrideAtCompileTime != Eigen::Dynamic - && std::is_constructible::value>; - template - using stride_ctor_inner - = bool_constant, stride_ctor_dual>::value - && S::InnerStrideAtCompileTime == Eigen::Dynamic - && S::OuterStrideAtCompileTime != Eigen::Dynamic - && std::is_constructible::value>; - - template ::value, int> = 0> - static S make_stride(EigenIndex, EigenIndex) { - return S(); - } - template ::value, int> = 0> - static S make_stride(EigenIndex outer, EigenIndex inner) { - return S(outer, inner); - } - template ::value, int> = 0> - static S make_stride(EigenIndex outer, EigenIndex) { - return S(outer); - } - template ::value, int> = 0> - static S make_stride(EigenIndex, EigenIndex inner) { - return S(inner); - } -}; - -// type_caster for special matrix types (e.g. DiagonalMatrix), which are EigenBase, but not -// EigenDense (i.e. they don't have a data(), at least not with the usual matrix layout). -// load() is not supported, but we can cast them into the python domain by first copying to a -// regular Eigen::Matrix, then casting that. -template -struct type_caster::value>> { -protected: - using Matrix - = Eigen::Matrix; - using props = EigenProps; - -public: - static handle cast(const Type &src, return_value_policy /* policy */, handle /* parent */) { - handle h = eigen_encapsulate(new Matrix(src)); - return h; - } - static handle cast(const Type *src, return_value_policy policy, handle parent) { - return cast(*src, policy, parent); - } - - static constexpr auto name = props::descriptor; - - // Explicitly delete these: support python -> C++ conversion on these (i.e. these can be return - // types but not bound arguments). We still provide them (with an explicitly delete) so that - // you end up here if you try anyway. - bool load(handle, bool) = delete; - operator Type() = delete; - template - using cast_op_type = Type; -}; - -template -struct type_caster::value>> { - using Scalar = typename Type::Scalar; - using StorageIndex = remove_reference_t().outerIndexPtr())>; - using Index = typename Type::Index; - static constexpr bool rowMajor = Type::IsRowMajor; - - bool load(handle src, bool) { - if (!src) { - return false; - } - - auto obj = reinterpret_borrow(src); - object sparse_module = module_::import("scipy.sparse"); - object matrix_type = sparse_module.attr(rowMajor ? "csr_matrix" : "csc_matrix"); - - if (!type::handle_of(obj).is(matrix_type)) { - try { - obj = matrix_type(obj); - } catch (const error_already_set &) { - return false; - } - } - - auto values = array_t((object) obj.attr("data")); - auto innerIndices = array_t((object) obj.attr("indices")); - auto outerIndices = array_t((object) obj.attr("indptr")); - auto shape = pybind11::tuple((pybind11::object) obj.attr("shape")); - auto nnz = obj.attr("nnz").cast(); - - if (!values || !innerIndices || !outerIndices) { - return false; - } - - value = EigenMapSparseMatrix(shape[0].cast(), - shape[1].cast(), - std::move(nnz), - outerIndices.mutable_data(), - innerIndices.mutable_data(), - values.mutable_data()); - - return true; - } - - static handle cast(const Type &src, return_value_policy /* policy */, handle /* parent */) { - const_cast(src).makeCompressed(); - - object matrix_type - = module_::import("scipy.sparse").attr(rowMajor ? "csr_matrix" : "csc_matrix"); - - array data(src.nonZeros(), src.valuePtr()); - array outerIndices((rowMajor ? src.rows() : src.cols()) + 1, src.outerIndexPtr()); - array innerIndices(src.nonZeros(), src.innerIndexPtr()); - - return matrix_type(pybind11::make_tuple( - std::move(data), std::move(innerIndices), std::move(outerIndices)), - pybind11::make_tuple(src.rows(), src.cols())) - .release(); - } - - PYBIND11_TYPE_CASTER(Type, - const_name<(Type::IsRowMajor) != 0>("scipy.sparse.csr_matrix[", - "scipy.sparse.csc_matrix[") - + npy_format_descriptor::name + const_name("]")); -}; - -PYBIND11_NAMESPACE_END(detail) -PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) +#include "eigen/matrix.h" diff --git a/pybind11/include/pybind11/eigen/common.h b/pybind11/include/pybind11/eigen/common.h new file mode 100644 index 0000000..24f56d1 --- /dev/null +++ b/pybind11/include/pybind11/eigen/common.h @@ -0,0 +1,9 @@ +// Copyright (c) 2023 The pybind Community. + +#pragma once + +// Common message for `static_assert()`s, which are useful to easily +// preempt much less obvious errors. +#define PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED \ + "Pointer types (in particular `PyObject *`) are not supported as scalar types for Eigen " \ + "types." diff --git a/pybind11/include/pybind11/eigen/matrix.h b/pybind11/include/pybind11/eigen/matrix.h new file mode 100644 index 0000000..8d4342f --- /dev/null +++ b/pybind11/include/pybind11/eigen/matrix.h @@ -0,0 +1,714 @@ +/* + pybind11/eigen/matrix.h: Transparent conversion for dense and sparse Eigen matrices + + Copyright (c) 2016 Wenzel Jakob + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#pragma once + +#include "../numpy.h" +#include "common.h" + +/* HINT: To suppress warnings originating from the Eigen headers, use -isystem. + See also: + https://stackoverflow.com/questions/2579576/i-dir-vs-isystem-dir + https://stackoverflow.com/questions/1741816/isystem-for-ms-visual-studio-c-compiler +*/ +PYBIND11_WARNING_PUSH +PYBIND11_WARNING_DISABLE_MSVC(5054) // https://github.com/pybind/pybind11/pull/3741 +// C5054: operator '&': deprecated between enumerations of different types +#if defined(__MINGW32__) +PYBIND11_WARNING_DISABLE_GCC("-Wmaybe-uninitialized") +#endif + +#include +#include + +PYBIND11_WARNING_POP + +// Eigen prior to 3.2.7 doesn't have proper move constructors--but worse, some classes get implicit +// move constructors that break things. We could detect this an explicitly copy, but an extra copy +// of matrices seems highly undesirable. +static_assert(EIGEN_VERSION_AT_LEAST(3, 2, 7), + "Eigen matrix support in pybind11 requires Eigen >= 3.2.7"); + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +PYBIND11_WARNING_DISABLE_MSVC(4127) + +// Provide a convenience alias for easier pass-by-ref usage with fully dynamic strides: +using EigenDStride = Eigen::Stride; +template +using EigenDRef = Eigen::Ref; +template +using EigenDMap = Eigen::Map; + +PYBIND11_NAMESPACE_BEGIN(detail) + +#if EIGEN_VERSION_AT_LEAST(3, 3, 0) +using EigenIndex = Eigen::Index; +template +using EigenMapSparseMatrix = Eigen::Map>; +#else +using EigenIndex = EIGEN_DEFAULT_DENSE_INDEX_TYPE; +template +using EigenMapSparseMatrix = Eigen::MappedSparseMatrix; +#endif + +// Matches Eigen::Map, Eigen::Ref, blocks, etc: +template +using is_eigen_dense_map = all_of, + std::is_base_of, T>>; +template +using is_eigen_mutable_map = std::is_base_of, T>; +template +using is_eigen_dense_plain + = all_of>, is_template_base_of>; +template +using is_eigen_sparse = is_template_base_of; +// Test for objects inheriting from EigenBase that aren't captured by the above. This +// basically covers anything that can be assigned to a dense matrix but that don't have a typical +// matrix data layout that can be copied from their .data(). For example, DiagonalMatrix and +// SelfAdjointView fall into this category. +template +using is_eigen_other + = all_of, + negation, is_eigen_dense_plain, is_eigen_sparse>>>; + +// Captures numpy/eigen conformability status (returned by EigenProps::conformable()): +template +struct EigenConformable { + bool conformable = false; + EigenIndex rows = 0, cols = 0; + EigenDStride stride{0, 0}; // Only valid if negativestrides is false! + bool negativestrides = false; // If true, do not use stride! + + // NOLINTNEXTLINE(google-explicit-constructor) + EigenConformable(bool fits = false) : conformable{fits} {} + // Matrix type: + EigenConformable(EigenIndex r, EigenIndex c, EigenIndex rstride, EigenIndex cstride) + : conformable{true}, rows{r}, cols{c}, + // TODO: when Eigen bug #747 is fixed, remove the tests for non-negativity. + // http://eigen.tuxfamily.org/bz/show_bug.cgi?id=747 + stride{EigenRowMajor ? (rstride > 0 ? rstride : 0) + : (cstride > 0 ? cstride : 0) /* outer stride */, + EigenRowMajor ? (cstride > 0 ? cstride : 0) + : (rstride > 0 ? rstride : 0) /* inner stride */}, + negativestrides{rstride < 0 || cstride < 0} {} + // Vector type: + EigenConformable(EigenIndex r, EigenIndex c, EigenIndex stride) + : EigenConformable(r, c, r == 1 ? c * stride : stride, c == 1 ? r : r * stride) {} + + template + bool stride_compatible() const { + // To have compatible strides, we need (on both dimensions) one of fully dynamic strides, + // matching strides, or a dimension size of 1 (in which case the stride value is + // irrelevant). Alternatively, if any dimension size is 0, the strides are not relevant + // (and numpy β‰₯ 1.23 sets the strides to 0 in that case, so we need to check explicitly). + if (negativestrides) { + return false; + } + if (rows == 0 || cols == 0) { + return true; + } + return (props::inner_stride == Eigen::Dynamic || props::inner_stride == stride.inner() + || (EigenRowMajor ? cols : rows) == 1) + && (props::outer_stride == Eigen::Dynamic || props::outer_stride == stride.outer() + || (EigenRowMajor ? rows : cols) == 1); + } + // NOLINTNEXTLINE(google-explicit-constructor) + operator bool() const { return conformable; } +}; + +template +struct eigen_extract_stride { + using type = Type; +}; +template +struct eigen_extract_stride> { + using type = StrideType; +}; +template +struct eigen_extract_stride> { + using type = StrideType; +}; + +// Helper struct for extracting information from an Eigen type +template +struct EigenProps { + using Type = Type_; + using Scalar = typename Type::Scalar; + using StrideType = typename eigen_extract_stride::type; + static constexpr EigenIndex rows = Type::RowsAtCompileTime, cols = Type::ColsAtCompileTime, + size = Type::SizeAtCompileTime; + static constexpr bool row_major = Type::IsRowMajor, + vector + = Type::IsVectorAtCompileTime, // At least one dimension has fixed size 1 + fixed_rows = rows != Eigen::Dynamic, fixed_cols = cols != Eigen::Dynamic, + fixed = size != Eigen::Dynamic, // Fully-fixed size + dynamic = !fixed_rows && !fixed_cols; // Fully-dynamic size + + template + using if_zero = std::integral_constant; + static constexpr EigenIndex inner_stride + = if_zero::value, + outer_stride = if_zero < StrideType::OuterStrideAtCompileTime, + vector ? size + : row_major ? cols + : rows > ::value; + static constexpr bool dynamic_stride + = inner_stride == Eigen::Dynamic && outer_stride == Eigen::Dynamic; + static constexpr bool requires_row_major + = !dynamic_stride && !vector && (row_major ? inner_stride : outer_stride) == 1; + static constexpr bool requires_col_major + = !dynamic_stride && !vector && (row_major ? outer_stride : inner_stride) == 1; + + // Takes an input array and determines whether we can make it fit into the Eigen type. If + // the array is a vector, we attempt to fit it into either an Eigen 1xN or Nx1 vector + // (preferring the latter if it will fit in either, i.e. for a fully dynamic matrix type). + static EigenConformable conformable(const array &a) { + const auto dims = a.ndim(); + if (dims < 1 || dims > 2) { + return false; + } + + if (dims == 2) { // Matrix type: require exact match (or dynamic) + + EigenIndex np_rows = a.shape(0), np_cols = a.shape(1), + np_rstride = a.strides(0) / static_cast(sizeof(Scalar)), + np_cstride = a.strides(1) / static_cast(sizeof(Scalar)); + if ((fixed_rows && np_rows != rows) || (fixed_cols && np_cols != cols)) { + return false; + } + + return {np_rows, np_cols, np_rstride, np_cstride}; + } + + // Otherwise we're storing an n-vector. Only one of the strides will be used, but + // whichever is used, we want the (single) numpy stride value. + const EigenIndex n = a.shape(0), + stride = a.strides(0) / static_cast(sizeof(Scalar)); + + if (vector) { // Eigen type is a compile-time vector + if (fixed && size != n) { + return false; // Vector size mismatch + } + return {rows == 1 ? 1 : n, cols == 1 ? 1 : n, stride}; + } + if (fixed) { + // The type has a fixed size, but is not a vector: abort + return false; + } + if (fixed_cols) { + // Since this isn't a vector, cols must be != 1. We allow this only if it exactly + // equals the number of elements (rows is Dynamic, and so 1 row is allowed). + if (cols != n) { + return false; + } + return {1, n, stride}; + } // Otherwise it's either fully dynamic, or column dynamic; both become a column vector + if (fixed_rows && rows != n) { + return false; + } + return {n, 1, stride}; + } + + static constexpr bool show_writeable + = is_eigen_dense_map::value && is_eigen_mutable_map::value; + static constexpr bool show_order = is_eigen_dense_map::value; + static constexpr bool show_c_contiguous = show_order && requires_row_major; + static constexpr bool show_f_contiguous + = !show_c_contiguous && show_order && requires_col_major; + + static constexpr auto descriptor + = const_name("numpy.ndarray[") + npy_format_descriptor::name + const_name("[") + + const_name(const_name<(size_t) rows>(), const_name("m")) + const_name(", ") + + const_name(const_name<(size_t) cols>(), const_name("n")) + const_name("]") + + + // For a reference type (e.g. Ref) we have other constraints that might need to + // be satisfied: writeable=True (for a mutable reference), and, depending on the map's + // stride options, possibly f_contiguous or c_contiguous. We include them in the + // descriptor output to provide some hint as to why a TypeError is occurring (otherwise + // it can be confusing to see that a function accepts a 'numpy.ndarray[float64[3,2]]' and + // an error message that you *gave* a numpy.ndarray of the right type and dimensions. + const_name(", flags.writeable", "") + + const_name(", flags.c_contiguous", "") + + const_name(", flags.f_contiguous", "") + const_name("]"); +}; + +// Casts an Eigen type to numpy array. If given a base, the numpy array references the src data, +// otherwise it'll make a copy. writeable lets you turn off the writeable flag for the array. +template +handle +eigen_array_cast(typename props::Type const &src, handle base = handle(), bool writeable = true) { + constexpr ssize_t elem_size = sizeof(typename props::Scalar); + array a; + if (props::vector) { + a = array({src.size()}, {elem_size * src.innerStride()}, src.data(), base); + } else { + a = array({src.rows(), src.cols()}, + {elem_size * src.rowStride(), elem_size * src.colStride()}, + src.data(), + base); + } + + if (!writeable) { + array_proxy(a.ptr())->flags &= ~detail::npy_api::NPY_ARRAY_WRITEABLE_; + } + + return a.release(); +} + +// Takes an lvalue ref to some Eigen type and a (python) base object, creating a numpy array that +// reference the Eigen object's data with `base` as the python-registered base class (if omitted, +// the base will be set to None, and lifetime management is up to the caller). The numpy array is +// non-writeable if the given type is const. +template +handle eigen_ref_array(Type &src, handle parent = none()) { + // none here is to get past array's should-we-copy detection, which currently always + // copies when there is no base. Setting the base to None should be harmless. + return eigen_array_cast(src, parent, !std::is_const::value); +} + +// Takes a pointer to some dense, plain Eigen type, builds a capsule around it, then returns a +// numpy array that references the encapsulated data with a python-side reference to the capsule to +// tie its destruction to that of any dependent python objects. Const-ness is determined by +// whether or not the Type of the pointer given is const. +template ::value>> +handle eigen_encapsulate(Type *src) { + capsule base(src, [](void *o) { delete static_cast(o); }); + return eigen_ref_array(*src, base); +} + +// Type caster for regular, dense matrix types (e.g. MatrixXd), but not maps/refs/etc. of dense +// types. +template +struct type_caster::value>> { + using Scalar = typename Type::Scalar; + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); + using props = EigenProps; + + bool load(handle src, bool convert) { + // If we're in no-convert mode, only load if given an array of the correct type + if (!convert && !isinstance>(src)) { + return false; + } + + // Coerce into an array, but don't do type conversion yet; the copy below handles it. + auto buf = array::ensure(src); + + if (!buf) { + return false; + } + + auto dims = buf.ndim(); + if (dims < 1 || dims > 2) { + return false; + } + + auto fits = props::conformable(buf); + if (!fits) { + return false; + } + + // Allocate the new type, then build a numpy reference into it + value = Type(fits.rows, fits.cols); + auto ref = reinterpret_steal(eigen_ref_array(value)); + if (dims == 1) { + ref = ref.squeeze(); + } else if (ref.ndim() == 1) { + buf = buf.squeeze(); + } + + int result = detail::npy_api::get().PyArray_CopyInto_(ref.ptr(), buf.ptr()); + + if (result < 0) { // Copy failed! + PyErr_Clear(); + return false; + } + + return true; + } + +private: + // Cast implementation + template + static handle cast_impl(CType *src, return_value_policy policy, handle parent) { + switch (policy) { + case return_value_policy::take_ownership: + case return_value_policy::automatic: + return eigen_encapsulate(src); + case return_value_policy::move: + return eigen_encapsulate(new CType(std::move(*src))); + case return_value_policy::copy: + return eigen_array_cast(*src); + case return_value_policy::reference: + case return_value_policy::automatic_reference: + return eigen_ref_array(*src); + case return_value_policy::reference_internal: + return eigen_ref_array(*src, parent); + default: + throw cast_error("unhandled return_value_policy: should not happen!"); + }; + } + +public: + // Normal returned non-reference, non-const value: + static handle cast(Type &&src, return_value_policy /* policy */, handle parent) { + return cast_impl(&src, return_value_policy::move, parent); + } + // If you return a non-reference const, we mark the numpy array readonly: + static handle cast(const Type &&src, return_value_policy /* policy */, handle parent) { + return cast_impl(&src, return_value_policy::move, parent); + } + // lvalue reference return; default (automatic) becomes copy + static handle cast(Type &src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic + || policy == return_value_policy::automatic_reference) { + policy = return_value_policy::copy; + } + return cast_impl(&src, policy, parent); + } + // const lvalue reference return; default (automatic) becomes copy + static handle cast(const Type &src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic + || policy == return_value_policy::automatic_reference) { + policy = return_value_policy::copy; + } + return cast(&src, policy, parent); + } + // non-const pointer return + static handle cast(Type *src, return_value_policy policy, handle parent) { + return cast_impl(src, policy, parent); + } + // const pointer return + static handle cast(const Type *src, return_value_policy policy, handle parent) { + return cast_impl(src, policy, parent); + } + + static constexpr auto name = props::descriptor; + + // NOLINTNEXTLINE(google-explicit-constructor) + operator Type *() { return &value; } + // NOLINTNEXTLINE(google-explicit-constructor) + operator Type &() { return value; } + // NOLINTNEXTLINE(google-explicit-constructor) + operator Type &&() && { return std::move(value); } + template + using cast_op_type = movable_cast_op_type; + +private: + Type value; +}; + +// Base class for casting reference/map/block/etc. objects back to python. +template +struct eigen_map_caster { + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); + +private: + using props = EigenProps; + +public: + // Directly referencing a ref/map's data is a bit dangerous (whatever the map/ref points to has + // to stay around), but we'll allow it under the assumption that you know what you're doing + // (and have an appropriate keep_alive in place). We return a numpy array pointing directly at + // the ref's data (The numpy array ends up read-only if the ref was to a const matrix type.) + // Note that this means you need to ensure you don't destroy the object in some other way (e.g. + // with an appropriate keep_alive, or with a reference to a statically allocated matrix). + static handle cast(const MapType &src, return_value_policy policy, handle parent) { + switch (policy) { + case return_value_policy::copy: + return eigen_array_cast(src); + case return_value_policy::reference_internal: + return eigen_array_cast(src, parent, is_eigen_mutable_map::value); + case return_value_policy::reference: + case return_value_policy::automatic: + case return_value_policy::automatic_reference: + return eigen_array_cast(src, none(), is_eigen_mutable_map::value); + default: + // move, take_ownership don't make any sense for a ref/map: + pybind11_fail("Invalid return_value_policy for Eigen Map/Ref/Block type"); + } + } + + static constexpr auto name = props::descriptor; + + // Explicitly delete these: support python -> C++ conversion on these (i.e. these can be return + // types but not bound arguments). We still provide them (with an explicitly delete) so that + // you end up here if you try anyway. + bool load(handle, bool) = delete; + operator MapType() = delete; + template + using cast_op_type = MapType; +}; + +// We can return any map-like object (but can only load Refs, specialized next): +template +struct type_caster::value>> : eigen_map_caster {}; + +// Loader for Ref<...> arguments. See the documentation for info on how to make this work without +// copying (it requires some extra effort in many cases). +template +struct type_caster< + Eigen::Ref, + enable_if_t>::value>> + : public eigen_map_caster> { +private: + using Type = Eigen::Ref; + using props = EigenProps; + using Scalar = typename props::Scalar; + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); + using MapType = Eigen::Map; + using Array + = array_t; + static constexpr bool need_writeable = is_eigen_mutable_map::value; + // Delay construction (these have no default constructor) + std::unique_ptr map; + std::unique_ptr ref; + // Our array. When possible, this is just a numpy array pointing to the source data, but + // sometimes we can't avoid copying (e.g. input is not a numpy array at all, has an + // incompatible layout, or is an array of a type that needs to be converted). Using a numpy + // temporary (rather than an Eigen temporary) saves an extra copy when we need both type + // conversion and storage order conversion. (Note that we refuse to use this temporary copy + // when loading an argument for a Ref with M non-const, i.e. a read-write reference). + Array copy_or_ref; + +public: + bool load(handle src, bool convert) { + // First check whether what we have is already an array of the right type. If not, we + // can't avoid a copy (because the copy is also going to do type conversion). + bool need_copy = !isinstance(src); + + EigenConformable fits; + if (!need_copy) { + // We don't need a converting copy, but we also need to check whether the strides are + // compatible with the Ref's stride requirements + auto aref = reinterpret_borrow(src); + + if (aref && (!need_writeable || aref.writeable())) { + fits = props::conformable(aref); + if (!fits) { + return false; // Incompatible dimensions + } + if (!fits.template stride_compatible()) { + need_copy = true; + } else { + copy_or_ref = std::move(aref); + } + } else { + need_copy = true; + } + } + + if (need_copy) { + // We need to copy: If we need a mutable reference, or we're not supposed to convert + // (either because we're in the no-convert overload pass, or because we're explicitly + // instructed not to copy (via `py::arg().noconvert()`) we have to fail loading. + if (!convert || need_writeable) { + return false; + } + + Array copy = Array::ensure(src); + if (!copy) { + return false; + } + fits = props::conformable(copy); + if (!fits || !fits.template stride_compatible()) { + return false; + } + copy_or_ref = std::move(copy); + loader_life_support::add_patient(copy_or_ref); + } + + ref.reset(); + map.reset(new MapType(data(copy_or_ref), + fits.rows, + fits.cols, + make_stride(fits.stride.outer(), fits.stride.inner()))); + ref.reset(new Type(*map)); + + return true; + } + + // NOLINTNEXTLINE(google-explicit-constructor) + operator Type *() { return ref.get(); } + // NOLINTNEXTLINE(google-explicit-constructor) + operator Type &() { return *ref; } + template + using cast_op_type = pybind11::detail::cast_op_type<_T>; + +private: + template ::value, int> = 0> + Scalar *data(Array &a) { + return a.mutable_data(); + } + + template ::value, int> = 0> + const Scalar *data(Array &a) { + return a.data(); + } + + // Attempt to figure out a constructor of `Stride` that will work. + // If both strides are fixed, use a default constructor: + template + using stride_ctor_default = bool_constant::value>; + // Otherwise, if there is a two-index constructor, assume it is (outer,inner) like + // Eigen::Stride, and use it: + template + using stride_ctor_dual + = bool_constant::value + && std::is_constructible::value>; + // Otherwise, if there is a one-index constructor, and just one of the strides is dynamic, use + // it (passing whichever stride is dynamic). + template + using stride_ctor_outer + = bool_constant, stride_ctor_dual>::value + && S::OuterStrideAtCompileTime == Eigen::Dynamic + && S::InnerStrideAtCompileTime != Eigen::Dynamic + && std::is_constructible::value>; + template + using stride_ctor_inner + = bool_constant, stride_ctor_dual>::value + && S::InnerStrideAtCompileTime == Eigen::Dynamic + && S::OuterStrideAtCompileTime != Eigen::Dynamic + && std::is_constructible::value>; + + template ::value, int> = 0> + static S make_stride(EigenIndex, EigenIndex) { + return S(); + } + template ::value, int> = 0> + static S make_stride(EigenIndex outer, EigenIndex inner) { + return S(outer, inner); + } + template ::value, int> = 0> + static S make_stride(EigenIndex outer, EigenIndex) { + return S(outer); + } + template ::value, int> = 0> + static S make_stride(EigenIndex, EigenIndex inner) { + return S(inner); + } +}; + +// type_caster for special matrix types (e.g. DiagonalMatrix), which are EigenBase, but not +// EigenDense (i.e. they don't have a data(), at least not with the usual matrix layout). +// load() is not supported, but we can cast them into the python domain by first copying to a +// regular Eigen::Matrix, then casting that. +template +struct type_caster::value>> { + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); + +protected: + using Matrix + = Eigen::Matrix; + using props = EigenProps; + +public: + static handle cast(const Type &src, return_value_policy /* policy */, handle /* parent */) { + handle h = eigen_encapsulate(new Matrix(src)); + return h; + } + static handle cast(const Type *src, return_value_policy policy, handle parent) { + return cast(*src, policy, parent); + } + + static constexpr auto name = props::descriptor; + + // Explicitly delete these: support python -> C++ conversion on these (i.e. these can be return + // types but not bound arguments). We still provide them (with an explicitly delete) so that + // you end up here if you try anyway. + bool load(handle, bool) = delete; + operator Type() = delete; + template + using cast_op_type = Type; +}; + +template +struct type_caster::value>> { + using Scalar = typename Type::Scalar; + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); + using StorageIndex = remove_reference_t().outerIndexPtr())>; + using Index = typename Type::Index; + static constexpr bool rowMajor = Type::IsRowMajor; + + bool load(handle src, bool) { + if (!src) { + return false; + } + + auto obj = reinterpret_borrow(src); + object sparse_module = module_::import("scipy.sparse"); + object matrix_type = sparse_module.attr(rowMajor ? "csr_matrix" : "csc_matrix"); + + if (!type::handle_of(obj).is(matrix_type)) { + try { + obj = matrix_type(obj); + } catch (const error_already_set &) { + return false; + } + } + + auto values = array_t((object) obj.attr("data")); + auto innerIndices = array_t((object) obj.attr("indices")); + auto outerIndices = array_t((object) obj.attr("indptr")); + auto shape = pybind11::tuple((pybind11::object) obj.attr("shape")); + auto nnz = obj.attr("nnz").cast(); + + if (!values || !innerIndices || !outerIndices) { + return false; + } + + value = EigenMapSparseMatrix(shape[0].cast(), + shape[1].cast(), + std::move(nnz), + outerIndices.mutable_data(), + innerIndices.mutable_data(), + values.mutable_data()); + + return true; + } + + static handle cast(const Type &src, return_value_policy /* policy */, handle /* parent */) { + const_cast(src).makeCompressed(); + + object matrix_type + = module_::import("scipy.sparse").attr(rowMajor ? "csr_matrix" : "csc_matrix"); + + array data(src.nonZeros(), src.valuePtr()); + array outerIndices((rowMajor ? src.rows() : src.cols()) + 1, src.outerIndexPtr()); + array innerIndices(src.nonZeros(), src.innerIndexPtr()); + + return matrix_type(pybind11::make_tuple( + std::move(data), std::move(innerIndices), std::move(outerIndices)), + pybind11::make_tuple(src.rows(), src.cols())) + .release(); + } + + PYBIND11_TYPE_CASTER(Type, + const_name<(Type::IsRowMajor) != 0>("scipy.sparse.csr_matrix[", + "scipy.sparse.csc_matrix[") + + npy_format_descriptor::name + const_name("]")); +}; + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/pybind11/include/pybind11/eigen/tensor.h b/pybind11/include/pybind11/eigen/tensor.h new file mode 100644 index 0000000..25d12ba --- /dev/null +++ b/pybind11/include/pybind11/eigen/tensor.h @@ -0,0 +1,516 @@ +/* + pybind11/eigen/tensor.h: Transparent conversion for Eigen tensors + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#pragma once + +#include "../numpy.h" +#include "common.h" + +#if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER) +static_assert(__GNUC__ > 5, "Eigen Tensor support in pybind11 requires GCC > 5.0"); +#endif + +// Disable warnings for Eigen +PYBIND11_WARNING_PUSH +PYBIND11_WARNING_DISABLE_MSVC(4554) +PYBIND11_WARNING_DISABLE_MSVC(4127) +#if defined(__MINGW32__) +PYBIND11_WARNING_DISABLE_GCC("-Wmaybe-uninitialized") +#endif + +#include + +PYBIND11_WARNING_POP + +static_assert(EIGEN_VERSION_AT_LEAST(3, 3, 0), + "Eigen Tensor support in pybind11 requires Eigen >= 3.3.0"); + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +PYBIND11_WARNING_DISABLE_MSVC(4127) + +PYBIND11_NAMESPACE_BEGIN(detail) + +inline bool is_tensor_aligned(const void *data) { + return (reinterpret_cast(data) % EIGEN_DEFAULT_ALIGN_BYTES) == 0; +} + +template +constexpr int compute_array_flag_from_tensor() { + static_assert((static_cast(T::Layout) == static_cast(Eigen::RowMajor)) + || (static_cast(T::Layout) == static_cast(Eigen::ColMajor)), + "Layout must be row or column major"); + return (static_cast(T::Layout) == static_cast(Eigen::RowMajor)) ? array::c_style + : array::f_style; +} + +template +struct eigen_tensor_helper {}; + +template +struct eigen_tensor_helper> { + using Type = Eigen::Tensor; + using ValidType = void; + + static Eigen::DSizes get_shape(const Type &f) { + return f.dimensions(); + } + + static constexpr bool + is_correct_shape(const Eigen::DSizes & /*shape*/) { + return true; + } + + template + struct helper {}; + + template + struct helper> { + static constexpr auto value = concat(const_name(((void) Is, "?"))...); + }; + + static constexpr auto dimensions_descriptor + = helper())>::value; + + template + static Type *alloc(Args &&...args) { + return new Type(std::forward(args)...); + } + + static void free(Type *tensor) { delete tensor; } +}; + +template +struct eigen_tensor_helper< + Eigen::TensorFixedSize, Options_, IndexType>> { + using Type = Eigen::TensorFixedSize, Options_, IndexType>; + using ValidType = void; + + static constexpr Eigen::DSizes + get_shape(const Type & /*f*/) { + return get_shape(); + } + + static constexpr Eigen::DSizes get_shape() { + return Eigen::DSizes(Indices...); + } + + static bool + is_correct_shape(const Eigen::DSizes &shape) { + return get_shape() == shape; + } + + static constexpr auto dimensions_descriptor = concat(const_name()...); + + template + static Type *alloc(Args &&...args) { + Eigen::aligned_allocator allocator; + return ::new (allocator.allocate(1)) Type(std::forward(args)...); + } + + static void free(Type *tensor) { + Eigen::aligned_allocator allocator; + tensor->~Type(); + allocator.deallocate(tensor, 1); + } +}; + +template +struct get_tensor_descriptor { + static constexpr auto details + = const_name(", flags.writeable", "") + + const_name(Type::Layout) == static_cast(Eigen::RowMajor)>( + ", flags.c_contiguous", ", flags.f_contiguous"); + static constexpr auto value + = const_name("numpy.ndarray[") + npy_format_descriptor::name + + const_name("[") + eigen_tensor_helper>::dimensions_descriptor + + const_name("]") + const_name(details, const_name("")) + const_name("]"); +}; + +// When EIGEN_AVOID_STL_ARRAY is defined, Eigen::DSizes does not have the begin() member +// function. Falling back to a simple loop works around this issue. +// +// We need to disable the type-limits warning for the inner loop when size = 0. + +PYBIND11_WARNING_PUSH +PYBIND11_WARNING_DISABLE_GCC("-Wtype-limits") + +template +std::vector convert_dsizes_to_vector(const Eigen::DSizes &arr) { + std::vector result(size); + + for (size_t i = 0; i < size; i++) { + result[i] = arr[i]; + } + + return result; +} + +template +Eigen::DSizes get_shape_for_array(const array &arr) { + Eigen::DSizes result; + const T *shape = arr.shape(); + for (size_t i = 0; i < size; i++) { + result[i] = shape[i]; + } + + return result; +} + +PYBIND11_WARNING_POP + +template +struct type_caster::ValidType> { + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); + using Helper = eigen_tensor_helper; + static constexpr auto temp_name = get_tensor_descriptor::value; + PYBIND11_TYPE_CASTER(Type, temp_name); + + bool load(handle src, bool convert) { + if (!convert) { + if (!isinstance(src)) { + return false; + } + array temp = array::ensure(src); + if (!temp) { + return false; + } + + if (!temp.dtype().is(dtype::of())) { + return false; + } + } + + array_t()> arr( + reinterpret_borrow(src)); + + if (arr.ndim() != Type::NumIndices) { + return false; + } + auto shape = get_shape_for_array(arr); + + if (!Helper::is_correct_shape(shape)) { + return false; + } + +#if EIGEN_VERSION_AT_LEAST(3, 4, 0) + auto data_pointer = arr.data(); +#else + // Handle Eigen bug + auto data_pointer = const_cast(arr.data()); +#endif + + if (is_tensor_aligned(arr.data())) { + value = Eigen::TensorMap(data_pointer, shape); + } else { + value = Eigen::TensorMap(data_pointer, shape); + } + + return true; + } + + static handle cast(Type &&src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::reference + || policy == return_value_policy::reference_internal) { + pybind11_fail("Cannot use a reference return value policy for an rvalue"); + } + return cast_impl(&src, return_value_policy::move, parent); + } + + static handle cast(const Type &&src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::reference + || policy == return_value_policy::reference_internal) { + pybind11_fail("Cannot use a reference return value policy for an rvalue"); + } + return cast_impl(&src, return_value_policy::move, parent); + } + + static handle cast(Type &src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic + || policy == return_value_policy::automatic_reference) { + policy = return_value_policy::copy; + } + return cast_impl(&src, policy, parent); + } + + static handle cast(const Type &src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic + || policy == return_value_policy::automatic_reference) { + policy = return_value_policy::copy; + } + return cast(&src, policy, parent); + } + + static handle cast(Type *src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic) { + policy = return_value_policy::take_ownership; + } else if (policy == return_value_policy::automatic_reference) { + policy = return_value_policy::reference; + } + return cast_impl(src, policy, parent); + } + + static handle cast(const Type *src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic) { + policy = return_value_policy::take_ownership; + } else if (policy == return_value_policy::automatic_reference) { + policy = return_value_policy::reference; + } + return cast_impl(src, policy, parent); + } + + template + static handle cast_impl(C *src, return_value_policy policy, handle parent) { + object parent_object; + bool writeable = false; + switch (policy) { + case return_value_policy::move: + if (std::is_const::value) { + pybind11_fail("Cannot move from a constant reference"); + } + + src = Helper::alloc(std::move(*src)); + + parent_object + = capsule(src, [](void *ptr) { Helper::free(reinterpret_cast(ptr)); }); + writeable = true; + break; + + case return_value_policy::take_ownership: + if (std::is_const::value) { + // This cast is ugly, and might be UB in some cases, but we don't have an + // alternative here as we must free that memory + Helper::free(const_cast(src)); + pybind11_fail("Cannot take ownership of a const reference"); + } + + parent_object + = capsule(src, [](void *ptr) { Helper::free(reinterpret_cast(ptr)); }); + writeable = true; + break; + + case return_value_policy::copy: + writeable = true; + break; + + case return_value_policy::reference: + parent_object = none(); + writeable = !std::is_const::value; + break; + + case return_value_policy::reference_internal: + // Default should do the right thing + if (!parent) { + pybind11_fail("Cannot use reference internal when there is no parent"); + } + parent_object = reinterpret_borrow(parent); + writeable = !std::is_const::value; + break; + + default: + pybind11_fail("pybind11 bug in eigen.h, please file a bug report"); + } + + auto result = array_t()>( + convert_dsizes_to_vector(Helper::get_shape(*src)), src->data(), parent_object); + + if (!writeable) { + array_proxy(result.ptr())->flags &= ~detail::npy_api::NPY_ARRAY_WRITEABLE_; + } + + return result.release(); + } +}; + +template = true> +StoragePointerType get_array_data_for_type(array &arr) { +#if EIGEN_VERSION_AT_LEAST(3, 4, 0) + return reinterpret_cast(arr.data()); +#else + // Handle Eigen bug + return reinterpret_cast(const_cast(arr.data())); +#endif +} + +template = true> +StoragePointerType get_array_data_for_type(array &arr) { + return reinterpret_cast(arr.mutable_data()); +} + +template +struct get_storage_pointer_type; + +template +struct get_storage_pointer_type> { + using SPT = typename MapType::StoragePointerType; +}; + +template +struct get_storage_pointer_type> { + using SPT = typename MapType::PointerArgType; +}; + +template +struct type_caster, + typename eigen_tensor_helper>::ValidType> { + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); + using MapType = Eigen::TensorMap; + using Helper = eigen_tensor_helper>; + + bool load(handle src, bool /*convert*/) { + // Note that we have a lot more checks here as we want to make sure to avoid copies + if (!isinstance(src)) { + return false; + } + auto arr = reinterpret_borrow(src); + if ((arr.flags() & compute_array_flag_from_tensor()) == 0) { + return false; + } + + if (!arr.dtype().is(dtype::of())) { + return false; + } + + if (arr.ndim() != Type::NumIndices) { + return false; + } + + constexpr bool is_aligned = (Options & Eigen::Aligned) != 0; + + if (is_aligned && !is_tensor_aligned(arr.data())) { + return false; + } + + auto shape = get_shape_for_array(arr); + + if (!Helper::is_correct_shape(shape)) { + return false; + } + + if (needs_writeable && !arr.writeable()) { + return false; + } + + auto result = get_array_data_for_type::SPT, + needs_writeable>(arr); + + value.reset(new MapType(std::move(result), std::move(shape))); + + return true; + } + + static handle cast(MapType &&src, return_value_policy policy, handle parent) { + return cast_impl(&src, policy, parent); + } + + static handle cast(const MapType &&src, return_value_policy policy, handle parent) { + return cast_impl(&src, policy, parent); + } + + static handle cast(MapType &src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic + || policy == return_value_policy::automatic_reference) { + policy = return_value_policy::copy; + } + return cast_impl(&src, policy, parent); + } + + static handle cast(const MapType &src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic + || policy == return_value_policy::automatic_reference) { + policy = return_value_policy::copy; + } + return cast(&src, policy, parent); + } + + static handle cast(MapType *src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic) { + policy = return_value_policy::take_ownership; + } else if (policy == return_value_policy::automatic_reference) { + policy = return_value_policy::reference; + } + return cast_impl(src, policy, parent); + } + + static handle cast(const MapType *src, return_value_policy policy, handle parent) { + if (policy == return_value_policy::automatic) { + policy = return_value_policy::take_ownership; + } else if (policy == return_value_policy::automatic_reference) { + policy = return_value_policy::reference; + } + return cast_impl(src, policy, parent); + } + + template + static handle cast_impl(C *src, return_value_policy policy, handle parent) { + object parent_object; + constexpr bool writeable = !std::is_const::value; + switch (policy) { + case return_value_policy::reference: + parent_object = none(); + break; + + case return_value_policy::reference_internal: + // Default should do the right thing + if (!parent) { + pybind11_fail("Cannot use reference internal when there is no parent"); + } + parent_object = reinterpret_borrow(parent); + break; + + case return_value_policy::take_ownership: + delete src; + // fallthrough + default: + // move, take_ownership don't make any sense for a ref/map: + pybind11_fail("Invalid return_value_policy for Eigen Map type, must be either " + "reference or reference_internal"); + } + + auto result = array_t()>( + convert_dsizes_to_vector(Helper::get_shape(*src)), + src->data(), + std::move(parent_object)); + + if (!writeable) { + array_proxy(result.ptr())->flags &= ~detail::npy_api::NPY_ARRAY_WRITEABLE_; + } + + return result.release(); + } + +#if EIGEN_VERSION_AT_LEAST(3, 4, 0) + + static constexpr bool needs_writeable = !std::is_const::SPT>::type>::value; +#else + // Handle Eigen bug + static constexpr bool needs_writeable = !std::is_const::value; +#endif + +protected: + // TODO: Move to std::optional once std::optional has more support + std::unique_ptr value; + +public: + static constexpr auto name = get_tensor_descriptor::value; + explicit operator MapType *() { return value.get(); } + explicit operator MapType &() { return *value; } + explicit operator MapType &&() && { return std::move(*value); } + + template + using cast_op_type = ::pybind11::detail::movable_cast_op_type; +}; + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/pybind11/include/pybind11/embed.h b/pybind11/include/pybind11/embed.h index d6999cd..caa14f4 100644 --- a/pybind11/include/pybind11/embed.h +++ b/pybind11/include/pybind11/embed.h @@ -86,38 +86,26 @@ inline wchar_t *widen_chars(const char *safe_arg) { return widened_arg; } -PYBIND11_NAMESPACE_END(detail) - -/** \rst - Initialize the Python interpreter. No other pybind11 or CPython API functions can be - called before this is done; with the exception of `PYBIND11_EMBEDDED_MODULE`. The - optional `init_signal_handlers` parameter can be used to skip the registration of - signal handlers (see the `Python documentation`_ for details). Calling this function - again after the interpreter has already been initialized is a fatal error. - - If initializing the Python interpreter fails, then the program is terminated. (This - is controlled by the CPython runtime and is an exception to pybind11's normal behavior - of throwing exceptions on errors.) - - The remaining optional parameters, `argc`, `argv`, and `add_program_dir_to_path` are - used to populate ``sys.argv`` and ``sys.path``. - See the |PySys_SetArgvEx documentation|_ for details. - - .. _Python documentation: https://docs.python.org/3/c-api/init.html#c.Py_InitializeEx - .. |PySys_SetArgvEx documentation| replace:: ``PySys_SetArgvEx`` documentation - .. _PySys_SetArgvEx documentation: https://docs.python.org/3/c-api/init.html#c.PySys_SetArgvEx - \endrst */ -inline void initialize_interpreter(bool init_signal_handlers = true, - int argc = 0, - const char *const *argv = nullptr, - bool add_program_dir_to_path = true) { +inline void precheck_interpreter() { if (Py_IsInitialized() != 0) { pybind11_fail("The interpreter is already running"); } +} -#if PY_VERSION_HEX < 0x030B0000 +#if !defined(PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX) +# define PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX (0x03080000) +#endif +#if PY_VERSION_HEX < PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX +inline void initialize_interpreter_pre_pyconfig(bool init_signal_handlers, + int argc, + const char *const *argv, + bool add_program_dir_to_path) { + detail::precheck_interpreter(); Py_InitializeEx(init_signal_handlers ? 1 : 0); +# if defined(WITH_THREAD) && PY_VERSION_HEX < 0x03070000 + PyEval_InitThreads(); +# endif // Before it was special-cased in python 3.8, passing an empty or null argv // caused a segfault, so we have to reimplement the special case ourselves. @@ -147,24 +135,30 @@ inline void initialize_interpreter(bool init_signal_handlers = true, auto *pysys_argv = widened_argv.get(); PySys_SetArgvEx(argc, pysys_argv, static_cast(add_program_dir_to_path)); -#else - PyConfig config; - PyConfig_InitIsolatedConfig(&config); - config.install_signal_handlers = init_signal_handlers ? 1 : 0; +} +#endif - PyStatus status = PyConfig_SetBytesArgv(&config, argc, const_cast(argv)); - if (PyStatus_Exception(status)) { +PYBIND11_NAMESPACE_END(detail) + +#if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX +inline void initialize_interpreter(PyConfig *config, + int argc = 0, + const char *const *argv = nullptr, + bool add_program_dir_to_path = true) { + detail::precheck_interpreter(); + PyStatus status = PyConfig_SetBytesArgv(config, argc, const_cast(argv)); + if (PyStatus_Exception(status) != 0) { // A failure here indicates a character-encoding failure or the python // interpreter out of memory. Give up. - PyConfig_Clear(&config); - throw std::runtime_error(PyStatus_IsError(status) ? status.err_msg - : "Failed to prepare CPython"); + PyConfig_Clear(config); + throw std::runtime_error(PyStatus_IsError(status) != 0 ? status.err_msg + : "Failed to prepare CPython"); } - status = Py_InitializeFromConfig(&config); - PyConfig_Clear(&config); - if (PyStatus_Exception(status)) { - throw std::runtime_error(PyStatus_IsError(status) ? status.err_msg - : "Failed to init CPython"); + status = Py_InitializeFromConfig(config); + if (PyStatus_Exception(status) != 0) { + PyConfig_Clear(config); + throw std::runtime_error(PyStatus_IsError(status) != 0 ? status.err_msg + : "Failed to init CPython"); } if (add_program_dir_to_path) { PyRun_SimpleString("import sys, os.path; " @@ -172,6 +166,44 @@ inline void initialize_interpreter(bool init_signal_handlers = true, "os.path.abspath(os.path.dirname(sys.argv[0])) " "if sys.argv and os.path.exists(sys.argv[0]) else '')"); } + PyConfig_Clear(config); +} +#endif + +/** \rst + Initialize the Python interpreter. No other pybind11 or CPython API functions can be + called before this is done; with the exception of `PYBIND11_EMBEDDED_MODULE`. The + optional `init_signal_handlers` parameter can be used to skip the registration of + signal handlers (see the `Python documentation`_ for details). Calling this function + again after the interpreter has already been initialized is a fatal error. + + If initializing the Python interpreter fails, then the program is terminated. (This + is controlled by the CPython runtime and is an exception to pybind11's normal behavior + of throwing exceptions on errors.) + + The remaining optional parameters, `argc`, `argv`, and `add_program_dir_to_path` are + used to populate ``sys.argv`` and ``sys.path``. + See the |PySys_SetArgvEx documentation|_ for details. + + .. _Python documentation: https://docs.python.org/3/c-api/init.html#c.Py_InitializeEx + .. |PySys_SetArgvEx documentation| replace:: ``PySys_SetArgvEx`` documentation + .. _PySys_SetArgvEx documentation: https://docs.python.org/3/c-api/init.html#c.PySys_SetArgvEx + \endrst */ +inline void initialize_interpreter(bool init_signal_handlers = true, + int argc = 0, + const char *const *argv = nullptr, + bool add_program_dir_to_path = true) { +#if PY_VERSION_HEX < PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX + detail::initialize_interpreter_pre_pyconfig( + init_signal_handlers, argc, argv, add_program_dir_to_path); +#else + PyConfig config; + PyConfig_InitPythonConfig(&config); + // See PR #4473 for background + config.parse_argv = 0; + + config.install_signal_handlers = init_signal_handlers ? 1 : 0; + initialize_interpreter(&config, argc, argv, add_program_dir_to_path); #endif } @@ -211,16 +243,14 @@ inline void initialize_interpreter(bool init_signal_handlers = true, \endrst */ inline void finalize_interpreter() { - handle builtins(PyEval_GetBuiltins()); - const char *id = PYBIND11_INTERNALS_ID; - // Get the internals pointer (without creating it if it doesn't exist). It's possible for the // internals to be created during Py_Finalize() (e.g. if a py::capsule calls `get_internals()` // during destruction), so we get the pointer-pointer here and check it after Py_Finalize(). detail::internals **internals_ptr_ptr = detail::get_internals_pp(); - // It could also be stashed in builtins, so look there too: - if (builtins.contains(id) && isinstance(builtins[id])) { - internals_ptr_ptr = capsule(builtins[id]); + // It could also be stashed in state_dict, so look there too: + if (object internals_obj + = get_internals_obj_from_state_dict(detail::get_python_state_dict())) { + internals_ptr_ptr = detail::get_internals_pp_from_capsule(internals_obj); } // Local internals contains data managed by the current interpreter, so we must clear them to // avoid undefined behaviors when initializing another interpreter @@ -259,6 +289,15 @@ class scoped_interpreter { initialize_interpreter(init_signal_handlers, argc, argv, add_program_dir_to_path); } +#if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX + explicit scoped_interpreter(PyConfig *config, + int argc = 0, + const char *const *argv = nullptr, + bool add_program_dir_to_path = true) { + initialize_interpreter(config, argc, argv, add_program_dir_to_path); + } +#endif + scoped_interpreter(const scoped_interpreter &) = delete; scoped_interpreter(scoped_interpreter &&other) noexcept { other.is_valid = false; } scoped_interpreter &operator=(const scoped_interpreter &) = delete; diff --git a/pybind11/include/pybind11/functional.h b/pybind11/include/pybind11/functional.h index 4034990..87ec4d1 100644 --- a/pybind11/include/pybind11/functional.h +++ b/pybind11/include/pybind11/functional.h @@ -48,9 +48,16 @@ struct type_caster> { */ if (auto cfunc = func.cpp_function()) { auto *cfunc_self = PyCFunction_GET_SELF(cfunc.ptr()); - if (isinstance(cfunc_self)) { + if (cfunc_self == nullptr) { + PyErr_Clear(); + } else if (isinstance(cfunc_self)) { auto c = reinterpret_borrow(cfunc_self); - auto *rec = (function_record *) c; + + function_record *rec = nullptr; + // Check that we can safely reinterpret the capsule into a function_record + if (detail::is_function_record_capsule(c)) { + rec = c.get_pointer(); + } while (rec != nullptr) { if (rec->is_stateless @@ -110,7 +117,7 @@ struct type_caster> { template static handle cast(Func &&f_, return_value_policy policy, handle /* parent */) { if (!f_) { - return none().inc_ref(); + return none().release(); } auto result = f_.template target(); diff --git a/pybind11/include/pybind11/gil.h b/pybind11/include/pybind11/gil.h index a0b5de1..570a558 100644 --- a/pybind11/include/pybind11/gil.h +++ b/pybind11/include/pybind11/gil.h @@ -10,7 +10,10 @@ #pragma once #include "detail/common.h" -#include "detail/internals.h" + +#if defined(WITH_THREAD) && !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) +# include "detail/internals.h" +#endif PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) @@ -21,7 +24,9 @@ PyThreadState *get_thread_state_unchecked(); PYBIND11_NAMESPACE_END(detail) -#if defined(WITH_THREAD) && !defined(PYPY_VERSION) +#if defined(WITH_THREAD) + +# if !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) /* The functions below essentially reproduce the PyGILState_* API using a RAII * pattern, but there are a few important differences: @@ -62,11 +67,11 @@ class gil_scoped_acquire { if (!tstate) { tstate = PyThreadState_New(internals.istate); -# if defined(PYBIND11_DETAILED_ERROR_MESSAGES) +# if defined(PYBIND11_DETAILED_ERROR_MESSAGES) if (!tstate) { pybind11_fail("scoped_acquire: could not create thread state!"); } -# endif +# endif tstate->gilstate_counter = 0; PYBIND11_TLS_REPLACE_VALUE(internals.tstate, tstate); } else { @@ -80,24 +85,27 @@ class gil_scoped_acquire { inc_ref(); } + gil_scoped_acquire(const gil_scoped_acquire &) = delete; + gil_scoped_acquire &operator=(const gil_scoped_acquire &) = delete; + void inc_ref() { ++tstate->gilstate_counter; } PYBIND11_NOINLINE void dec_ref() { --tstate->gilstate_counter; -# if defined(PYBIND11_DETAILED_ERROR_MESSAGES) +# if defined(PYBIND11_DETAILED_ERROR_MESSAGES) if (detail::get_thread_state_unchecked() != tstate) { pybind11_fail("scoped_acquire::dec_ref(): thread state must be current!"); } if (tstate->gilstate_counter < 0) { pybind11_fail("scoped_acquire::dec_ref(): reference count underflow!"); } -# endif +# endif if (tstate->gilstate_counter == 0) { -# if defined(PYBIND11_DETAILED_ERROR_MESSAGES) +# if defined(PYBIND11_DETAILED_ERROR_MESSAGES) if (!release) { pybind11_fail("scoped_acquire::dec_ref(): internal error!"); } -# endif +# endif PyThreadState_Clear(tstate); if (active) { PyThreadState_DeleteCurrent(); @@ -144,6 +152,9 @@ class gil_scoped_release { } } + gil_scoped_release(const gil_scoped_release &) = delete; + gil_scoped_release &operator=(const gil_scoped_release &) = delete; + /// This method will disable the PyThreadState_DeleteCurrent call and the /// GIL won't be acquired. This method should be used if the interpreter /// could be shutting down when this is called, as thread deletion is not @@ -172,12 +183,16 @@ class gil_scoped_release { bool disassoc; bool active = true; }; -#elif defined(PYPY_VERSION) + +# else // PYBIND11_SIMPLE_GIL_MANAGEMENT + class gil_scoped_acquire { PyGILState_STATE state; public: - gil_scoped_acquire() { state = PyGILState_Ensure(); } + gil_scoped_acquire() : state{PyGILState_Ensure()} {} + gil_scoped_acquire(const gil_scoped_acquire &) = delete; + gil_scoped_acquire &operator=(const gil_scoped_acquire &) = delete; ~gil_scoped_acquire() { PyGILState_Release(state); } void disarm() {} }; @@ -186,17 +201,39 @@ class gil_scoped_release { PyThreadState *state; public: - gil_scoped_release() { state = PyEval_SaveThread(); } + gil_scoped_release() : state{PyEval_SaveThread()} {} + gil_scoped_release(const gil_scoped_release &) = delete; + gil_scoped_release &operator=(const gil_scoped_release &) = delete; ~gil_scoped_release() { PyEval_RestoreThread(state); } void disarm() {} }; -#else + +# endif // PYBIND11_SIMPLE_GIL_MANAGEMENT + +#else // WITH_THREAD + class gil_scoped_acquire { +public: + gil_scoped_acquire() { + // Trick to suppress `unused variable` error messages (at call sites). + (void) (this != (this + 1)); + } + gil_scoped_acquire(const gil_scoped_acquire &) = delete; + gil_scoped_acquire &operator=(const gil_scoped_acquire &) = delete; void disarm() {} }; + class gil_scoped_release { +public: + gil_scoped_release() { + // Trick to suppress `unused variable` error messages (at call sites). + (void) (this != (this + 1)); + } + gil_scoped_release(const gil_scoped_release &) = delete; + gil_scoped_release &operator=(const gil_scoped_release &) = delete; void disarm() {} }; -#endif + +#endif // WITH_THREAD PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/pybind11/include/pybind11/numpy.h b/pybind11/include/pybind11/numpy.h index 0291b02..36077ec 100644 --- a/pybind11/include/pybind11/numpy.h +++ b/pybind11/include/pybind11/numpy.h @@ -36,6 +36,8 @@ static_assert(std::is_signed::value, "Py_intptr_t must be signed"); PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_WARNING_DISABLE_MSVC(4127) + class array; // Forward declaration PYBIND11_NAMESPACE_BEGIN(detail) @@ -537,7 +539,7 @@ PYBIND11_NAMESPACE_END(detail) class dtype : public object { public: - PYBIND11_OBJECT_DEFAULT(dtype, object, detail::npy_api::get().PyArrayDescr_Check_); + PYBIND11_OBJECT_DEFAULT(dtype, object, detail::npy_api::get().PyArrayDescr_Check_) explicit dtype(const buffer_info &info) { dtype descr(_dtype_from_pep3118()(pybind11::str(info.format))); @@ -562,6 +564,8 @@ class dtype : public object { m_ptr = from_args(args).release().ptr(); } + /// Return dtype for the given typenum (one of the NPY_TYPES). + /// https://numpy.org/devdocs/reference/c-api/array.html#c.PyArray_DescrFromType explicit dtype(int typenum) : object(detail::npy_api::get().PyArray_DescrFromType_(typenum), stolen_t{}) { if (m_ptr == nullptr) { @@ -875,7 +879,7 @@ class array : public buffer { */ template detail::unchecked_mutable_reference mutable_unchecked() & { - if (PYBIND11_SILENCE_MSVC_C4127(Dims >= 0) && ndim() != Dims) { + if (Dims >= 0 && ndim() != Dims) { throw std::domain_error("array has incorrect number of dimensions: " + std::to_string(ndim()) + "; expected " + std::to_string(Dims)); @@ -893,7 +897,7 @@ class array : public buffer { */ template detail::unchecked_reference unchecked() const & { - if (PYBIND11_SILENCE_MSVC_C4127(Dims >= 0) && ndim() != Dims) { + if (Dims >= 0 && ndim() != Dims) { throw std::domain_error("array has incorrect number of dimensions: " + std::to_string(ndim()) + "; expected " + std::to_string(Dims)); @@ -1119,10 +1123,10 @@ class array_t : public array { /** * Returns a proxy object that provides const access to the array's data without bounds or - * dimensionality checking. Unlike `unchecked()`, this does not require that the underlying - * array have the `writable` flag. Use with care: the array must not be destroyed or reshaped - * for the duration of the returned object, and the caller must take care not to access invalid - * dimensions or dimension indices. + * dimensionality checking. Unlike `mutable_unchecked()`, this does not require that the + * underlying array have the `writable` flag. Use with care: the array must not be destroyed + * or reshaped for the duration of the returned object, and the caller must take care not to + * access invalid dimensions or dimension indices. */ template detail::unchecked_reference unchecked() const & { @@ -1281,12 +1285,16 @@ struct npy_format_descriptor< public: static constexpr int value = values[detail::is_fmt_numeric::index]; - static pybind11::dtype dtype() { - if (auto *ptr = npy_api::get().PyArray_DescrFromType_(value)) { - return reinterpret_steal(ptr); - } - pybind11_fail("Unsupported buffer format!"); - } + static pybind11::dtype dtype() { return pybind11::dtype(/*typenum*/ value); } +}; + +template +struct npy_format_descriptor::value>> { + static constexpr auto name = const_name("object"); + + static constexpr int value = npy_api::NPY_OBJECT_; + + static pybind11::dtype dtype() { return pybind11::dtype(/*typenum*/ value); } }; #define PYBIND11_DECL_CHAR_FMT \ @@ -1401,7 +1409,7 @@ PYBIND11_NOINLINE void register_structured_dtype(any_container oss << '}'; auto format_str = oss.str(); - // Sanity check: verify that NumPy properly parses our buffer format string + // Smoke test: verify that NumPy properly parses our buffer format string auto &api = npy_api::get(); auto arr = array(buffer_info(nullptr, itemsize, format_str, 1)); if (!api.PyArray_EquivTypes_(dtype_ptr, arr.dtype().ptr())) { @@ -1469,7 +1477,7 @@ struct npy_format_descriptor { } // Extract name, offset and format descriptor for a struct field -# define PYBIND11_FIELD_DESCRIPTOR(T, Field) PYBIND11_FIELD_DESCRIPTOR_EX(T, Field, # Field) +# define PYBIND11_FIELD_DESCRIPTOR(T, Field) PYBIND11_FIELD_DESCRIPTOR_EX(T, Field, #Field) // The main idea of this macro is borrowed from https://github.com/swansontec/map-macro // (C) William Swanson, Paul Fultz @@ -1866,8 +1874,13 @@ struct vectorize_helper { auto result = returned_array::create(trivial, shape); + PYBIND11_WARNING_PUSH +#ifdef PYBIND11_DETECTED_CLANG_WITH_MISLEADING_CALL_STD_MOVE_EXPLICITLY_WARNING + PYBIND11_WARNING_DISABLE_CLANG("-Wreturn-std-move") +#endif + if (size == 0) { - return std::move(result); + return result; } /* Call the function */ @@ -1878,7 +1891,8 @@ struct vectorize_helper { apply_trivial(buffers, params, mutable_data, size, i_seq, vi_seq, bi_seq); } - return std::move(result); + return result; + PYBIND11_WARNING_POP } template diff --git a/pybind11/include/pybind11/operators.h b/pybind11/include/pybind11/operators.h index a0c3b78..16a88ae 100644 --- a/pybind11/include/pybind11/operators.h +++ b/pybind11/include/pybind11/operators.h @@ -84,6 +84,7 @@ struct op_impl {}; /// Operator implementation generator template struct op_ { + static constexpr bool op_enable_if_hook = true; template void execute(Class &cl, const Extra &...extra) const { using Base = typename Class::type; diff --git a/pybind11/include/pybind11/options.h b/pybind11/include/pybind11/options.h index 1e493bd..1b21225 100644 --- a/pybind11/include/pybind11/options.h +++ b/pybind11/include/pybind11/options.h @@ -47,6 +47,16 @@ class options { return *this; } + options &disable_enum_members_docstring() & { + global_state().show_enum_members_docstring = false; + return *this; + } + + options &enable_enum_members_docstring() & { + global_state().show_enum_members_docstring = true; + return *this; + } + // Getter methods (return the global state): static bool show_user_defined_docstrings() { @@ -55,6 +65,10 @@ class options { static bool show_function_signatures() { return global_state().show_function_signatures; } + static bool show_enum_members_docstring() { + return global_state().show_enum_members_docstring; + } + // This type is not meant to be allocated on the heap. void *operator new(size_t) = delete; @@ -63,6 +77,8 @@ class options { bool show_user_defined_docstrings = true; //< Include user-supplied texts in docstrings. bool show_function_signatures = true; //< Include auto-generated function signatures // in docstrings. + bool show_enum_members_docstring = true; //< Include auto-generated member list in enum + // docstrings. }; static state &global_state() { diff --git a/pybind11/include/pybind11/pybind11.h b/pybind11/include/pybind11/pybind11.h index d61dcd5..3bce1a0 100644 --- a/pybind11/include/pybind11/pybind11.h +++ b/pybind11/include/pybind11/pybind11.h @@ -35,6 +35,8 @@ # include #endif +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + /* https://stackoverflow.com/questions/46798456/handling-gccs-noexcept-type-warning This warning is about ABI compatibility, not code health. It is only actually needed in a couple places, but apparently GCC 7 "generates this warning if @@ -43,11 +45,10 @@ No other GCC version generates this warning. */ #if defined(__GNUC__) && __GNUC__ == 7 -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wnoexcept-type" +PYBIND11_WARNING_DISABLE_GCC("-Wnoexcept-type") #endif -PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_WARNING_DISABLE_MSVC(4127) PYBIND11_NAMESPACE_BEGIN(detail) @@ -83,6 +84,7 @@ class cpp_function : public function { cpp_function() = default; // NOLINTNEXTLINE(google-explicit-constructor) cpp_function(std::nullptr_t) {} + cpp_function(std::nullptr_t, const is_setter &) {} /// Construct a cpp_function from a vanilla function pointer template @@ -177,22 +179,22 @@ class cpp_function : public function { auto *rec = unique_rec.get(); /* Store the capture object directly in the function record if there is enough space */ - if (PYBIND11_SILENCE_MSVC_C4127(sizeof(capture) <= sizeof(rec->data))) { + if (sizeof(capture) <= sizeof(rec->data)) { /* Without these pragmas, GCC warns that there might not be enough space to use the placement new operator. However, the 'if' statement above ensures that this is the case. */ -#if defined(__GNUG__) && __GNUC__ >= 6 && !defined(__clang__) && !defined(__INTEL_COMPILER) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wplacement-new" + PYBIND11_WARNING_PUSH + +#if defined(__GNUG__) && __GNUC__ >= 6 + PYBIND11_WARNING_DISABLE_GCC("-Wplacement-new") #endif + new ((capture *) &rec->data) capture{std::forward(f)}; -#if defined(__GNUG__) && __GNUC__ >= 6 && !defined(__clang__) && !defined(__INTEL_COMPILER) -# pragma GCC diagnostic pop -#endif -#if defined(__GNUG__) && !PYBIND11_HAS_STD_LAUNDER && !defined(__INTEL_COMPILER) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wstrict-aliasing" + +#if !PYBIND11_HAS_STD_LAUNDER + PYBIND11_WARNING_DISABLE_GCC("-Wstrict-aliasing") #endif + // UB without std::launder, but without breaking ABI and/or // a significant refactoring it's "impossible" to solve. if (!std::is_trivially_destructible::value) { @@ -202,9 +204,7 @@ class cpp_function : public function { data->~capture(); }; } -#if defined(__GNUG__) && !PYBIND11_HAS_STD_LAUNDER && !defined(__INTEL_COMPILER) -# pragma GCC diagnostic pop -#endif + PYBIND11_WARNING_POP } else { rec->data[0] = new capture{std::forward(f)}; rec->free_data = [](function_record *r) { delete ((capture *) r->data[0]); }; @@ -245,10 +245,16 @@ class cpp_function : public function { using Guard = extract_guard_t; /* Perform the function call */ - handle result - = cast_out::cast(std::move(args_converter).template call(cap->f), - policy, - call.parent); + handle result; + if (call.func.is_setter) { + (void) std::move(args_converter).template call(cap->f); + result = none().release(); + } else { + result = cast_out::cast( + std::move(args_converter).template call(cap->f), + policy, + call.parent); + } /* Invoke call policy post-call hook */ process_attributes::postcall(call, result); @@ -468,13 +474,20 @@ class cpp_function : public function { if (rec->sibling) { if (PyCFunction_Check(rec->sibling.ptr())) { auto *self = PyCFunction_GET_SELF(rec->sibling.ptr()); - capsule rec_capsule = isinstance(self) ? reinterpret_borrow(self) - : capsule(self); - chain = (detail::function_record *) rec_capsule; - /* Never append a method to an overload chain of a parent class; - instead, hide the parent's overloads in this case */ - if (!chain->scope.is(rec->scope)) { + if (!isinstance(self)) { chain = nullptr; + } else { + auto rec_capsule = reinterpret_borrow(self); + if (detail::is_function_record_capsule(rec_capsule)) { + chain = rec_capsule.get_pointer(); + /* Never append a method to an overload chain of a parent class; + instead, hide the parent's overloads in this case */ + if (!chain->scope.is(rec->scope)) { + chain = nullptr; + } + } else { + chain = nullptr; + } } } // Don't trigger for things like the default __init__, which are wrapper_descriptors @@ -495,6 +508,7 @@ class cpp_function : public function { rec->def->ml_flags = METH_VARARGS | METH_KEYWORDS; capsule rec_capsule(unique_rec.release(), + detail::get_function_record_capsule_name(), [](void *ptr) { destruct((detail::function_record *) ptr); }); guarded_strdup.release(); @@ -661,10 +675,13 @@ class cpp_function : public function { /// Main dispatch logic for calls to functions bound using pybind11 static PyObject *dispatcher(PyObject *self, PyObject *args_in, PyObject *kwargs_in) { using namespace detail; + assert(isinstance(self)); /* Iterator over the list of potentially admissible overloads */ - const function_record *overloads = (function_record *) PyCapsule_GetPointer(self, nullptr), + const function_record *overloads = reinterpret_cast( + PyCapsule_GetPointer(self, get_function_record_capsule_name())), *it = overloads; + assert(overloads != nullptr); /* Need to know how many arguments + keyword arguments there are to pick the right overload */ @@ -1416,9 +1433,9 @@ template ::value, int> = 0> void call_operator_delete(T *p, size_t, size_t) { T::operator delete(p); } -template < - typename T, - enable_if_t::value && has_operator_delete_size::value, int> = 0> +template ::value && has_operator_delete_size::value, int> + = 0> void call_operator_delete(T *p, size_t s, size_t) { T::operator delete(p, s); } @@ -1578,14 +1595,14 @@ class class_ : public detail::generic_type { return *this; } - template - class_ &def(const detail::op_ &op, const Extra &...extra) { + template = 0> + class_ &def(const T &op, const Extra &...extra) { op.execute(*this, extra...); return *this; } - template - class_ &def_cast(const detail::op_ &op, const Extra &...extra) { + template = 0> + class_ &def_cast(const T &op, const Extra &...extra) { op.execute_cast(*this, extra...); return *this; } @@ -1719,7 +1736,8 @@ class class_ : public detail::generic_type { template class_ & def_property(const char *name, const Getter &fget, const Setter &fset, const Extra &...extra) { - return def_property(name, fget, cpp_function(method_adaptor(fset)), extra...); + return def_property( + name, fget, cpp_function(method_adaptor(fset), is_setter()), extra...); } template class_ &def_property(const char *name, @@ -1830,8 +1848,7 @@ class class_ : public detail::generic_type { if (holder_ptr) { init_holder_from_existing(v_h, holder_ptr, std::is_copy_constructible()); v_h.set_holder_constructed(); - } else if (PYBIND11_SILENCE_MSVC_C4127(detail::always_construct_holder::value) - || inst->owned) { + } else if (detail::always_construct_holder::value || inst->owned) { new (std::addressof(v_h.holder())) holder_type(v_h.value_ptr()); v_h.set_holder_constructed(); } @@ -1871,9 +1888,22 @@ class class_ : public detail::generic_type { static detail::function_record *get_function_record(handle h) { h = detail::get_function(h); - return h ? (detail::function_record *) reinterpret_borrow( - PyCFunction_GET_SELF(h.ptr())) - : nullptr; + if (!h) { + return nullptr; + } + + handle func_self = PyCFunction_GET_SELF(h.ptr()); + if (!func_self) { + throw error_already_set(); + } + if (!isinstance(func_self)) { + return nullptr; + } + auto cap = reinterpret_borrow(func_self); + if (!detail::is_function_record_capsule(cap)) { + return nullptr; + } + return cap.get_pointer(); } }; @@ -1950,29 +1980,35 @@ struct enum_base { name("name"), is_method(m_base)); - m_base.attr("__doc__") = static_property( - cpp_function( - [](handle arg) -> std::string { - std::string docstring; - dict entries = arg.attr("__entries"); - if (((PyTypeObject *) arg.ptr())->tp_doc) { - docstring += std::string(((PyTypeObject *) arg.ptr())->tp_doc) + "\n\n"; - } - docstring += "Members:"; - for (auto kv : entries) { - auto key = std::string(pybind11::str(kv.first)); - auto comment = kv.second[int_(1)]; - docstring += "\n\n " + key; - if (!comment.is_none()) { - docstring += " : " + (std::string) pybind11::str(comment); + if (options::show_enum_members_docstring()) { + m_base.attr("__doc__") = static_property( + cpp_function( + [](handle arg) -> std::string { + std::string docstring; + dict entries = arg.attr("__entries"); + if (((PyTypeObject *) arg.ptr())->tp_doc) { + docstring += std::string( + reinterpret_cast(arg.ptr())->tp_doc); + docstring += "\n\n"; } - } - return docstring; - }, - name("__doc__")), - none(), - none(), - ""); + docstring += "Members:"; + for (auto kv : entries) { + auto key = std::string(pybind11::str(kv.first)); + auto comment = kv.second[int_(1)]; + docstring += "\n\n "; + docstring += key; + if (!comment.is_none()) { + docstring += " : "; + docstring += pybind11::str(comment).cast(); + } + } + return docstring; + }, + name("__doc__")), + none(), + none(), + ""); + } m_base.attr("__members__") = static_property(cpp_function( [](handle arg) -> dict { @@ -2073,7 +2109,7 @@ struct enum_base { + "\" already exists!"); } - entries[name] = std::make_pair(value, doc); + entries[name] = pybind11::make_tuple(value, doc); m_base.attr(std::move(name)) = std::move(value); } @@ -2333,7 +2369,7 @@ template -iterator make_iterator_impl(Iterator &&first, Sentinel &&last, Extra &&...extra) { +iterator make_iterator_impl(Iterator first, Sentinel last, Extra &&...extra) { using state = detail::iterator_state; // TODO: state captures only the types of Extra, not the values @@ -2359,7 +2395,7 @@ iterator make_iterator_impl(Iterator &&first, Sentinel &&last, Extra &&...extra) Policy); } - return cast(state{std::forward(first), std::forward(last), true}); + return cast(state{first, last, true}); } PYBIND11_NAMESPACE_END(detail) @@ -2370,15 +2406,13 @@ template ::result_type, typename... Extra> -iterator make_iterator(Iterator &&first, Sentinel &&last, Extra &&...extra) { +iterator make_iterator(Iterator first, Sentinel last, Extra &&...extra) { return detail::make_iterator_impl, Policy, Iterator, Sentinel, ValueType, - Extra...>(std::forward(first), - std::forward(last), - std::forward(extra)...); + Extra...>(first, last, std::forward(extra)...); } /// Makes a python iterator over the keys (`.first`) of a iterator over pairs from a @@ -2388,15 +2422,13 @@ template ::result_type, typename... Extra> -iterator make_key_iterator(Iterator &&first, Sentinel &&last, Extra &&...extra) { +iterator make_key_iterator(Iterator first, Sentinel last, Extra &&...extra) { return detail::make_iterator_impl, Policy, Iterator, Sentinel, KeyType, - Extra...>(std::forward(first), - std::forward(last), - std::forward(extra)...); + Extra...>(first, last, std::forward(extra)...); } /// Makes a python iterator over the values (`.second`) of a iterator over pairs from a @@ -2406,15 +2438,13 @@ template ::result_type, typename... Extra> -iterator make_value_iterator(Iterator &&first, Sentinel &&last, Extra &&...extra) { +iterator make_value_iterator(Iterator first, Sentinel last, Extra &&...extra) { return detail::make_iterator_impl, Policy, Iterator, Sentinel, ValueType, - Extra...>(std::forward(first), - std::forward(last), - std::forward(extra)...); + Extra...>(first, last, std::forward(extra)...); } /// Makes an iterator over values of an stl container or other container supporting @@ -2858,7 +2888,3 @@ inline function get_overload(const T *this_ptr, const char *name) { PYBIND11_OVERRIDE_PURE(PYBIND11_TYPE(ret_type), PYBIND11_TYPE(cname), fn, __VA_ARGS__); PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) - -#if defined(__GNUC__) && __GNUC__ == 7 -# pragma GCC diagnostic pop // -Wnoexcept-type -#endif diff --git a/pybind11/include/pybind11/pytypes.h b/pybind11/include/pybind11/pytypes.h index 339b096..64aad63 100644 --- a/pybind11/include/pybind11/pytypes.h +++ b/pybind11/include/pybind11/pytypes.h @@ -33,6 +33,8 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_WARNING_DISABLE_MSVC(4127) + /* A few forward declarations */ class handle; class object; @@ -155,23 +157,23 @@ class object_api : public pyobject_tag { object operator-() const; object operator~() const; object operator+(object_api const &other) const; - object operator+=(object_api const &other) const; + object operator+=(object_api const &other); object operator-(object_api const &other) const; - object operator-=(object_api const &other) const; + object operator-=(object_api const &other); object operator*(object_api const &other) const; - object operator*=(object_api const &other) const; + object operator*=(object_api const &other); object operator/(object_api const &other) const; - object operator/=(object_api const &other) const; + object operator/=(object_api const &other); object operator|(object_api const &other) const; - object operator|=(object_api const &other) const; + object operator|=(object_api const &other); object operator&(object_api const &other) const; - object operator&=(object_api const &other) const; + object operator&=(object_api const &other); object operator^(object_api const &other) const; - object operator^=(object_api const &other) const; + object operator^=(object_api const &other); object operator<<(object_api const &other) const; - object operator<<=(object_api const &other) const; + object operator<<=(object_api const &other); object operator>>(object_api const &other) const; - object operator>>=(object_api const &other) const; + object operator>>=(object_api const &other); PYBIND11_DEPRECATED("Use py::str(obj) instead") pybind11::str str() const; @@ -230,7 +232,8 @@ class handle : public detail::object_api { detail::enable_if_t, detail::is_pyobj_ptr_or_nullptr_t>, std::is_convertible>::value, - int> = 0> + int> + = 0> // NOLINTNEXTLINE(google-explicit-constructor) handle(T &obj) : m_ptr(obj) {} @@ -246,6 +249,11 @@ class handle : public detail::object_api { const handle &inc_ref() const & { #ifdef PYBIND11_HANDLE_REF_DEBUG inc_ref_counter(1); +#endif +#ifdef PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF + if (m_ptr != nullptr && !PyGILState_Check()) { + throw_gilstate_error("pybind11::handle::inc_ref()"); + } #endif Py_XINCREF(m_ptr); return *this; @@ -257,6 +265,11 @@ class handle : public detail::object_api { this function automatically. Returns a reference to itself. \endrst */ const handle &dec_ref() const & { +#ifdef PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF + if (m_ptr != nullptr && !PyGILState_Check()) { + throw_gilstate_error("pybind11::handle::dec_ref()"); + } +#endif Py_XDECREF(m_ptr); return *this; } @@ -283,8 +296,33 @@ class handle : public detail::object_api { protected: PyObject *m_ptr = nullptr; -#ifdef PYBIND11_HANDLE_REF_DEBUG private: +#ifdef PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF + void throw_gilstate_error(const std::string &function_name) const { + fprintf( + stderr, + "%s is being called while the GIL is either not held or invalid. Please see " + "https://pybind11.readthedocs.io/en/stable/advanced/" + "misc.html#common-sources-of-global-interpreter-lock-errors for debugging advice.\n" + "If you are convinced there is no bug in your code, you can #define " + "PYBIND11_NO_ASSERT_GIL_HELD_INCREF_DECREF" + "to disable this check. In that case you have to ensure this #define is consistently " + "used for all translation units linked into a given pybind11 extension, otherwise " + "there will be ODR violations.", + function_name.c_str()); + fflush(stderr); + if (Py_TYPE(m_ptr)->tp_name != nullptr) { + fprintf(stderr, + "The failing %s call was triggered on a %s object.\n", + function_name.c_str(), + Py_TYPE(m_ptr)->tp_name); + fflush(stderr); + } + throw std::runtime_error(function_name + " PyGILState_Check() failure."); + } +#endif + +#ifdef PYBIND11_HANDLE_REF_DEBUG static std::size_t inc_ref_counter(std::size_t add) { thread_local std::size_t counter = 0; counter += add; @@ -334,12 +372,15 @@ class object : public handle { } object &operator=(const object &other) { - other.inc_ref(); - // Use temporary variable to ensure `*this` remains valid while - // `Py_XDECREF` executes, in case `*this` is accessible from Python. - handle temp(m_ptr); - m_ptr = other.m_ptr; - temp.dec_ref(); + // Skip inc_ref and dec_ref if both objects are the same + if (!this->is(other)) { + other.inc_ref(); + // Use temporary variable to ensure `*this` remains valid while + // `Py_XDECREF` executes, in case `*this` is accessible from Python. + handle temp(m_ptr); + m_ptr = other.m_ptr; + temp.dec_ref(); + } return *this; } @@ -353,6 +394,20 @@ class object : public handle { return *this; } +#define PYBIND11_INPLACE_OP(iop) \ + object iop(object_api const &other) { return operator=(handle::iop(other)); } + + PYBIND11_INPLACE_OP(operator+=) + PYBIND11_INPLACE_OP(operator-=) + PYBIND11_INPLACE_OP(operator*=) + PYBIND11_INPLACE_OP(operator/=) + PYBIND11_INPLACE_OP(operator|=) + PYBIND11_INPLACE_OP(operator&=) + PYBIND11_INPLACE_OP(operator^=) + PYBIND11_INPLACE_OP(operator<<=) + PYBIND11_INPLACE_OP(operator>>=) +#undef PYBIND11_INPLACE_OP + // Calling cast() on an object lvalue just copies (via handle::cast) template T cast() const &; @@ -413,7 +468,7 @@ PYBIND11_NAMESPACE_BEGIN(detail) // Equivalent to obj.__class__.__name__ (or obj.__name__ if obj is a class). inline const char *obj_class_name(PyObject *obj) { - if (Py_TYPE(obj) == &PyType_Type) { + if (PyType_Check(obj)) { return reinterpret_cast(obj)->tp_name; } return Py_TYPE(obj)->tp_name; @@ -421,13 +476,24 @@ inline const char *obj_class_name(PyObject *obj) { std::string error_string(); +// The code in this struct is very unusual, to minimize the chances of +// masking bugs (elsewhere) by errors during the error handling (here). +// This is meant to be a lifeline for troubleshooting long-running processes +// that crash under conditions that are virtually impossible to reproduce. +// Low-level implementation alternatives are preferred to higher-level ones +// that might raise cascading exceptions. Last-ditch-kind-of attempts are made +// to report as much of the original error as possible, even if there are +// secondary issues obtaining some of the details. struct error_fetch_and_normalize { - // Immediate normalization is long-established behavior (starting with - // https://github.com/pybind/pybind11/commit/135ba8deafb8bf64a15b24d1513899eb600e2011 - // from Sep 2016) and safest. Normalization could be deferred, but this could mask - // errors elsewhere, the performance gain is very minor in typical situations - // (usually the dominant bottleneck is EH unwinding), and the implementation here - // would be more complex. + // This comment only applies to Python <= 3.11: + // Immediate normalization is long-established behavior (starting with + // https://github.com/pybind/pybind11/commit/135ba8deafb8bf64a15b24d1513899eb600e2011 + // from Sep 2016) and safest. Normalization could be deferred, but this could mask + // errors elsewhere, the performance gain is very minor in typical situations + // (usually the dominant bottleneck is EH unwinding), and the implementation here + // would be more complex. + // Starting with Python 3.12, PyErr_Fetch() normalizes exceptions immediately. + // Any errors during normalization are tracked under __notes__. explicit error_fetch_and_normalize(const char *called) { PyErr_Fetch(&m_type.ptr(), &m_value.ptr(), &m_trace.ptr()); if (!m_type) { @@ -442,6 +508,14 @@ struct error_fetch_and_normalize { "of the original active exception type."); } m_lazy_error_string = exc_type_name_orig; +#if PY_VERSION_HEX >= 0x030C0000 + // The presence of __notes__ is likely due to exception normalization + // errors, although that is not necessarily true, therefore insert a + // hint only: + if (PyObject_HasAttrString(m_value.ptr(), "__notes__")) { + m_lazy_error_string += "[WITH __notes__]"; + } +#else // PyErr_NormalizeException() may change the exception type if there are cascading // failures. This can potentially be extremely confusing. PyErr_NormalizeException(&m_type.ptr(), &m_value.ptr(), &m_trace.ptr()); @@ -451,11 +525,17 @@ struct error_fetch_and_normalize { "active exception."); } const char *exc_type_name_norm = detail::obj_class_name(m_type.ptr()); - if (exc_type_name_orig == nullptr) { + if (exc_type_name_norm == nullptr) { pybind11_fail("Internal error: " + std::string(called) + " failed to obtain the name " "of the normalized active exception type."); } +# if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x07030a00 + // This behavior runs the risk of masking errors in the error handling, but avoids a + // conflict with PyPy, which relies on the normalization here to change OSError to + // FileNotFoundError (https://github.com/pybind/pybind11/issues/4075). + m_lazy_error_string = exc_type_name_norm; +# else if (exc_type_name_norm != m_lazy_error_string) { std::string msg = std::string(called) + ": MISMATCH of original and normalized " @@ -467,6 +547,8 @@ struct error_fetch_and_normalize { msg += ": " + format_value_and_trace(); pybind11_fail(msg); } +# endif +#endif } error_fetch_and_normalize(const error_fetch_and_normalize &) = delete; @@ -477,12 +559,64 @@ struct error_fetch_and_normalize { std::string message_error_string; if (m_value) { auto value_str = reinterpret_steal(PyObject_Str(m_value.ptr())); + constexpr const char *message_unavailable_exc + = ""; if (!value_str) { message_error_string = detail::error_string(); - result = ""; + result = message_unavailable_exc; + } else { + // Not using `value_str.cast()`, to not potentially throw a secondary + // error_already_set that will then result in process termination (#4288). + auto value_bytes = reinterpret_steal( + PyUnicode_AsEncodedString(value_str.ptr(), "utf-8", "backslashreplace")); + if (!value_bytes) { + message_error_string = detail::error_string(); + result = message_unavailable_exc; + } else { + char *buffer = nullptr; + Py_ssize_t length = 0; + if (PyBytes_AsStringAndSize(value_bytes.ptr(), &buffer, &length) == -1) { + message_error_string = detail::error_string(); + result = message_unavailable_exc; + } else { + result = std::string(buffer, static_cast(length)); + } + } + } +#if PY_VERSION_HEX >= 0x030B0000 + auto notes + = reinterpret_steal(PyObject_GetAttrString(m_value.ptr(), "__notes__")); + if (!notes) { + PyErr_Clear(); // No notes is good news. } else { - result = value_str.cast(); + auto len_notes = PyList_Size(notes.ptr()); + if (len_notes < 0) { + result += "\nFAILURE obtaining len(__notes__): " + detail::error_string(); + } else { + result += "\n__notes__ (len=" + std::to_string(len_notes) + "):"; + for (ssize_t i = 0; i < len_notes; i++) { + PyObject *note = PyList_GET_ITEM(notes.ptr(), i); + auto note_bytes = reinterpret_steal( + PyUnicode_AsEncodedString(note, "utf-8", "backslashreplace")); + if (!note_bytes) { + result += "\nFAILURE obtaining __notes__[" + std::to_string(i) + + "]: " + detail::error_string(); + } else { + char *buffer = nullptr; + Py_ssize_t length = 0; + if (PyBytes_AsStringAndSize(note_bytes.ptr(), &buffer, &length) + == -1) { + result += "\nFAILURE formatting __notes__[" + std::to_string(i) + + "]: " + detail::error_string(); + } else { + result += '\n'; + result += std::string(buffer, static_cast(length)); + } + } + } + } } +#endif } else { result = ""; } @@ -581,12 +715,6 @@ inline std::string error_string() { PYBIND11_NAMESPACE_END(detail) -#if defined(_MSC_VER) -# pragma warning(push) -# pragma warning(disable : 4275 4251) -// warning C4275: An exported class was derived from a class that wasn't exported. -// Can be ignored when derived from a STL class. -#endif /// Fetch and hold an error which was already set in Python. An instance of this is typically /// thrown to propagate python-side errors back through C++ which can either be caught manually or /// else falls back to the function dispatcher (which then raises the captured error back to @@ -646,9 +774,6 @@ class PYBIND11_EXPORT_EXCEPTION error_already_set : public std::exception { /// crashes (undefined behavior) if the Python interpreter is finalizing. static void m_fetched_error_deleter(detail::error_fetch_and_normalize *raw_ptr); }; -#if defined(_MSC_VER) -# pragma warning(pop) -#endif /// Replaces the current Python error indicator with the chosen error, performing a /// 'raise from' to indicate that the chosen error was caused by the original error. @@ -851,10 +976,8 @@ object object_or_cast(T &&o); // Match a PyObject*, which we want to convert directly to handle via its converting constructor inline handle object_or_cast(PyObject *ptr) { return ptr; } -#if defined(_MSC_VER) && _MSC_VER < 1920 -# pragma warning(push) -# pragma warning(disable : 4522) // warning C4522: multiple assignment operators specified -#endif +PYBIND11_WARNING_PUSH +PYBIND11_WARNING_DISABLE_MSVC(4522) // warning C4522: multiple assignment operators specified template class accessor : public object_api> { using key_type = typename Policy::key_type; @@ -918,9 +1041,7 @@ class accessor : public object_api> { key_type key; mutable object cache; }; -#if defined(_MSC_VER) && _MSC_VER < 1920 -# pragma warning(pop) -#endif +PYBIND11_WARNING_POP PYBIND11_NAMESPACE_BEGIN(accessor_policies) struct obj_attr { @@ -1266,7 +1387,7 @@ public: #define PYBIND11_OBJECT_CVT_DEFAULT(Name, Parent, CheckFun, ConvertFun) \ PYBIND11_OBJECT_CVT(Name, Parent, CheckFun, ConvertFun) \ - Name() : Parent() {} + Name() = default; #define PYBIND11_OBJECT_CHECK_FAILED(Name, o_ptr) \ ::pybind11::type_error("Object of type '" \ @@ -1289,7 +1410,7 @@ public: #define PYBIND11_OBJECT_DEFAULT(Name, Parent, CheckFun) \ PYBIND11_OBJECT(Name, Parent, CheckFun) \ - Name() : Parent() {} + Name() = default; /// \addtogroup pytypes /// @{ @@ -1358,7 +1479,7 @@ class iterator : public object { private: void advance() { value = reinterpret_steal(PyIter_Next(m_ptr)); - if (PyErr_Occurred()) { + if (value.ptr() == nullptr && PyErr_Occurred()) { throw error_already_set(); } } @@ -1408,6 +1529,9 @@ class str : public object { str(const char *c, const SzType &n) : object(PyUnicode_FromStringAndSize(c, ssize_t_cast(n)), stolen_t{}) { if (!m_ptr) { + if (PyErr_Occurred()) { + throw error_already_set(); + } pybind11_fail("Could not allocate string object!"); } } @@ -1417,6 +1541,9 @@ class str : public object { // NOLINTNEXTLINE(google-explicit-constructor) str(const char *c = "") : object(PyUnicode_FromString(c), stolen_t{}) { if (!m_ptr) { + if (PyErr_Occurred()) { + throw error_already_set(); + } pybind11_fail("Could not allocate string object!"); } } @@ -1574,6 +1701,9 @@ inline str::str(const bytes &b) { } auto obj = reinterpret_steal(PyUnicode_FromStringAndSize(buffer, length)); if (!obj) { + if (PyErr_Occurred()) { + throw error_already_set(); + } pybind11_fail("Could not allocate string object!"); } m_ptr = obj.release().ptr(); @@ -1651,7 +1781,7 @@ PYBIND11_NAMESPACE_BEGIN(detail) // unsigned type: (A)-1 != (B)-1 when A and B are unsigned types of different sizes). template Unsigned as_unsigned(PyObject *o) { - if (PYBIND11_SILENCE_MSVC_C4127(sizeof(Unsigned) <= sizeof(unsigned long))) { + if (sizeof(Unsigned) <= sizeof(unsigned long)) { unsigned long v = PyLong_AsUnsignedLong(o); return v == (unsigned long) -1 && PyErr_Occurred() ? (Unsigned) -1 : (Unsigned) v; } @@ -1668,7 +1798,7 @@ class int_ : public object { template ::value, int> = 0> // NOLINTNEXTLINE(google-explicit-constructor) int_(T value) { - if (PYBIND11_SILENCE_MSVC_C4127(sizeof(T) <= sizeof(long))) { + if (sizeof(T) <= sizeof(long)) { if (std::is_signed::value) { m_ptr = PyLong_FromLong((long) value); } else { @@ -1785,43 +1915,28 @@ class capsule : public object { explicit capsule(const void *value, const char *name = nullptr, - void (*destructor)(PyObject *) = nullptr) + PyCapsule_Destructor destructor = nullptr) : object(PyCapsule_New(const_cast(value), name, destructor), stolen_t{}) { if (!m_ptr) { throw error_already_set(); } } - PYBIND11_DEPRECATED("Please pass a destructor that takes a void pointer as input") - capsule(const void *value, void (*destruct)(PyObject *)) - : object(PyCapsule_New(const_cast(value), nullptr, destruct), stolen_t{}) { + PYBIND11_DEPRECATED("Please use the ctor with value, name, destructor args") + capsule(const void *value, PyCapsule_Destructor destructor) + : object(PyCapsule_New(const_cast(value), nullptr, destructor), stolen_t{}) { if (!m_ptr) { throw error_already_set(); } } + /// Capsule name is nullptr. capsule(const void *value, void (*destructor)(void *)) { - m_ptr = PyCapsule_New(const_cast(value), nullptr, [](PyObject *o) { - // guard if destructor called while err indicator is set - error_scope error_guard; - auto destructor = reinterpret_cast(PyCapsule_GetContext(o)); - if (destructor == nullptr) { - if (PyErr_Occurred()) { - throw error_already_set(); - } - pybind11_fail("Unable to get capsule context"); - } - const char *name = get_name_in_error_scope(o); - void *ptr = PyCapsule_GetPointer(o, name); - if (ptr == nullptr) { - throw error_already_set(); - } - destructor(ptr); - }); + initialize_with_void_ptr_destructor(value, nullptr, destructor); + } - if (!m_ptr || PyCapsule_SetContext(m_ptr, (void *) destructor) != 0) { - throw error_already_set(); - } + capsule(const void *value, const char *name, void (*destructor)(void *)) { + initialize_with_void_ptr_destructor(value, name, destructor); } explicit capsule(void (*destructor)()) { @@ -1889,6 +2004,32 @@ class capsule : public object { return name; } + + void initialize_with_void_ptr_destructor(const void *value, + const char *name, + void (*destructor)(void *)) { + m_ptr = PyCapsule_New(const_cast(value), name, [](PyObject *o) { + // guard if destructor called while err indicator is set + error_scope error_guard; + auto destructor = reinterpret_cast(PyCapsule_GetContext(o)); + if (destructor == nullptr && PyErr_Occurred()) { + throw error_already_set(); + } + const char *name = get_name_in_error_scope(o); + void *ptr = PyCapsule_GetPointer(o, name); + if (ptr == nullptr) { + throw error_already_set(); + } + + if (destructor != nullptr) { + destructor(ptr); + } + }); + + if (!m_ptr || PyCapsule_SetContext(m_ptr, reinterpret_cast(destructor)) != 0) { + throw error_already_set(); + } + } }; class tuple : public object { @@ -1943,7 +2084,11 @@ class dict : public object { void clear() /* py-non-const */ { PyDict_Clear(ptr()); } template bool contains(T &&key) const { - return PyDict_Contains(m_ptr, detail::object_or_cast(std::forward(key)).ptr()) == 1; + auto result = PyDict_Contains(m_ptr, detail::object_or_cast(std::forward(key)).ptr()); + if (result == -1) { + throw error_already_set(); + } + return result == 1; } private: @@ -1998,14 +2143,20 @@ class list : public object { detail::list_iterator end() const { return {*this, PyList_GET_SIZE(m_ptr)}; } template void append(T &&val) /* py-non-const */ { - PyList_Append(m_ptr, detail::object_or_cast(std::forward(val)).ptr()); + if (PyList_Append(m_ptr, detail::object_or_cast(std::forward(val)).ptr()) != 0) { + throw error_already_set(); + } } template ::value, int> = 0> void insert(const IdxType &index, ValType &&val) /* py-non-const */ { - PyList_Insert( - m_ptr, ssize_t_cast(index), detail::object_or_cast(std::forward(val)).ptr()); + if (PyList_Insert(m_ptr, + ssize_t_cast(index), + detail::object_or_cast(std::forward(val)).ptr()) + != 0) { + throw error_already_set(); + } } }; @@ -2023,7 +2174,11 @@ class anyset : public object { bool empty() const { return size() == 0; } template bool contains(T &&val) const { - return PySet_Contains(m_ptr, detail::object_or_cast(std::forward(val)).ptr()) == 1; + auto result = PySet_Contains(m_ptr, detail::object_or_cast(std::forward(val)).ptr()); + if (result == -1) { + throw error_already_set(); + } + return result == 1; } }; @@ -2364,29 +2519,39 @@ bool object_api::rich_compare(object_api const &other, int value) const { return result; \ } +#define PYBIND11_MATH_OPERATOR_BINARY_INPLACE(iop, fn) \ + template \ + object object_api::iop(object_api const &other) { \ + object result = reinterpret_steal(fn(derived().ptr(), other.derived().ptr())); \ + if (!result.ptr()) \ + throw error_already_set(); \ + return result; \ + } + PYBIND11_MATH_OPERATOR_UNARY(operator~, PyNumber_Invert) PYBIND11_MATH_OPERATOR_UNARY(operator-, PyNumber_Negative) PYBIND11_MATH_OPERATOR_BINARY(operator+, PyNumber_Add) -PYBIND11_MATH_OPERATOR_BINARY(operator+=, PyNumber_InPlaceAdd) +PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator+=, PyNumber_InPlaceAdd) PYBIND11_MATH_OPERATOR_BINARY(operator-, PyNumber_Subtract) -PYBIND11_MATH_OPERATOR_BINARY(operator-=, PyNumber_InPlaceSubtract) +PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator-=, PyNumber_InPlaceSubtract) PYBIND11_MATH_OPERATOR_BINARY(operator*, PyNumber_Multiply) -PYBIND11_MATH_OPERATOR_BINARY(operator*=, PyNumber_InPlaceMultiply) +PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator*=, PyNumber_InPlaceMultiply) PYBIND11_MATH_OPERATOR_BINARY(operator/, PyNumber_TrueDivide) -PYBIND11_MATH_OPERATOR_BINARY(operator/=, PyNumber_InPlaceTrueDivide) +PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator/=, PyNumber_InPlaceTrueDivide) PYBIND11_MATH_OPERATOR_BINARY(operator|, PyNumber_Or) -PYBIND11_MATH_OPERATOR_BINARY(operator|=, PyNumber_InPlaceOr) +PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator|=, PyNumber_InPlaceOr) PYBIND11_MATH_OPERATOR_BINARY(operator&, PyNumber_And) -PYBIND11_MATH_OPERATOR_BINARY(operator&=, PyNumber_InPlaceAnd) +PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator&=, PyNumber_InPlaceAnd) PYBIND11_MATH_OPERATOR_BINARY(operator^, PyNumber_Xor) -PYBIND11_MATH_OPERATOR_BINARY(operator^=, PyNumber_InPlaceXor) +PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator^=, PyNumber_InPlaceXor) PYBIND11_MATH_OPERATOR_BINARY(operator<<, PyNumber_Lshift) -PYBIND11_MATH_OPERATOR_BINARY(operator<<=, PyNumber_InPlaceLshift) +PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator<<=, PyNumber_InPlaceLshift) PYBIND11_MATH_OPERATOR_BINARY(operator>>, PyNumber_Rshift) -PYBIND11_MATH_OPERATOR_BINARY(operator>>=, PyNumber_InPlaceRshift) +PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator>>=, PyNumber_InPlaceRshift) #undef PYBIND11_MATH_OPERATOR_UNARY #undef PYBIND11_MATH_OPERATOR_BINARY +#undef PYBIND11_MATH_OPERATOR_BINARY_INPLACE PYBIND11_NAMESPACE_END(detail) PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/pybind11/include/pybind11/stl.h b/pybind11/include/pybind11/stl.h index ab30eca..f39f44f 100644 --- a/pybind11/include/pybind11/stl.h +++ b/pybind11/include/pybind11/stl.h @@ -45,21 +45,35 @@ using forwarded_type = conditional_t::value, /// Forwards a value U as rvalue or lvalue according to whether T is rvalue or lvalue; typically /// used for forwarding a container's elements. template -forwarded_type forward_like(U &&u) { +constexpr forwarded_type forward_like(U &&u) { return std::forward>(std::forward(u)); } +// Checks if a container has a STL style reserve method. +// This will only return true for a `reserve()` with a `void` return. +template +using has_reserve_method = std::is_same().reserve(0)), void>; + template struct set_caster { using type = Type; using key_conv = make_caster; +private: + template ::value, int> = 0> + void reserve_maybe(const anyset &s, Type *) { + value.reserve(s.size()); + } + void reserve_maybe(const anyset &, void *) {} + +public: bool load(handle src, bool convert) { if (!isinstance(src)) { return false; } auto s = reinterpret_borrow(src); value.clear(); + reserve_maybe(s, &value); for (auto entry : s) { key_conv conv; if (!conv.load(entry, convert)) { @@ -78,7 +92,7 @@ struct set_caster { pybind11::set s; for (auto &&value : src) { auto value_ = reinterpret_steal( - key_conv::cast(forward_like(value), policy, parent)); + key_conv::cast(detail::forward_like(value), policy, parent)); if (!value_ || !s.add(std::move(value_))) { return handle(); } @@ -94,12 +108,21 @@ struct map_caster { using key_conv = make_caster; using value_conv = make_caster; +private: + template ::value, int> = 0> + void reserve_maybe(const dict &d, Type *) { + value.reserve(d.size()); + } + void reserve_maybe(const dict &, void *) {} + +public: bool load(handle src, bool convert) { if (!isinstance(src)) { return false; } auto d = reinterpret_borrow(src); value.clear(); + reserve_maybe(d, &value); for (auto it : d) { key_conv kconv; value_conv vconv; @@ -122,9 +145,9 @@ struct map_caster { } for (auto &&kv : src) { auto key = reinterpret_steal( - key_conv::cast(forward_like(kv.first), policy_key, parent)); + key_conv::cast(detail::forward_like(kv.first), policy_key, parent)); auto value = reinterpret_steal( - value_conv::cast(forward_like(kv.second), policy_value, parent)); + value_conv::cast(detail::forward_like(kv.second), policy_value, parent)); if (!key || !value) { return handle(); } @@ -160,9 +183,7 @@ struct list_caster { } private: - template < - typename T = Type, - enable_if_t().reserve(0)), void>::value, int> = 0> + template ::value, int> = 0> void reserve_maybe(const sequence &s, Type *) { value.reserve(s.size()); } @@ -178,7 +199,7 @@ struct list_caster { ssize_t index = 0; for (auto &&value : src) { auto value_ = reinterpret_steal( - value_conv::cast(forward_like(value), policy, parent)); + value_conv::cast(detail::forward_like(value), policy, parent)); if (!value_) { return handle(); } @@ -242,7 +263,7 @@ struct array_caster { ssize_t index = 0; for (auto &&value : src) { auto value_ = reinterpret_steal( - value_conv::cast(forward_like(value), policy, parent)); + value_conv::cast(detail::forward_like(value), policy, parent)); if (!value_) { return handle(); } @@ -252,11 +273,11 @@ struct array_caster { } PYBIND11_TYPE_CASTER(ArrayType, - const_name("List[") + value_conv::name + const_name(const_name(""), const_name("Annotated[")) + + const_name("List[") + value_conv::name + const_name("]") + const_name(const_name(""), - const_name("[") + const_name() - + const_name("]")) - + const_name("]")); + const_name(", FixedSize(") + + const_name() + const_name(")]"))); }; template @@ -290,11 +311,12 @@ struct optional_caster { template static handle cast(T &&src, return_value_policy policy, handle parent) { if (!src) { - return none().inc_ref(); + return none().release(); } if (!std::is_lvalue_reference::value) { policy = return_value_policy_override::policy(policy); } + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) return value_conv::cast(*std::forward(src), policy, parent); } diff --git a/pybind11/include/pybind11/stl_bind.h b/pybind11/include/pybind11/stl_bind.h index 22a29b4..49f1b77 100644 --- a/pybind11/include/pybind11/stl_bind.h +++ b/pybind11/include/pybind11/stl_bind.h @@ -10,10 +10,13 @@ #pragma once #include "detail/common.h" +#include "detail/type_caster_base.h" +#include "cast.h" #include "operators.h" #include #include +#include PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) @@ -58,9 +61,11 @@ struct is_comparable< /* For a vector/map data structure, recursively check the value type (which is std::pair for maps) */ template -struct is_comparable::is_vector>> { - static constexpr const bool value = is_comparable::value; -}; +struct is_comparable::is_vector>> + : is_comparable::type_to_check_recursively> {}; + +template <> +struct is_comparable : std::true_type {}; /* For pairs, recursively check the two data types */ template @@ -352,13 +357,17 @@ void vector_accessor(enable_if_t::value, Class_> &cl) using DiffType = typename Vector::difference_type; using ItType = typename Vector::iterator; cl.def("__getitem__", [](const Vector &v, DiffType i) -> T { - if (i < 0 && (i += v.size()) < 0) { - throw index_error(); + if (i < 0) { + i += v.size(); + if (i < 0) { + throw index_error(); + } } - if ((SizeType) i >= v.size()) { + auto i_st = static_cast(i); + if (i_st >= v.size()) { throw index_error(); } - return v[(SizeType) i]; + return v[i_st]; }); cl.def( @@ -636,18 +645,52 @@ auto map_if_insertion_operator(Class_ &cl, std::string const &name) "Return the canonical string representation of this map."); } -template +template struct keys_view { - Map ↦ + virtual size_t len() = 0; + virtual iterator iter() = 0; + virtual bool contains(const KeyType &k) = 0; + virtual bool contains(const object &k) = 0; + virtual ~keys_view() = default; }; -template +template struct values_view { - Map ↦ + virtual size_t len() = 0; + virtual iterator iter() = 0; + virtual ~values_view() = default; }; -template +template struct items_view { + virtual size_t len() = 0; + virtual iterator iter() = 0; + virtual ~items_view() = default; +}; + +template +struct KeysViewImpl : public KeysView { + explicit KeysViewImpl(Map &map) : map(map) {} + size_t len() override { return map.size(); } + iterator iter() override { return make_key_iterator(map.begin(), map.end()); } + bool contains(const typename Map::key_type &k) override { return map.find(k) != map.end(); } + bool contains(const object &) override { return false; } + Map ↦ +}; + +template +struct ValuesViewImpl : public ValuesView { + explicit ValuesViewImpl(Map &map) : map(map) {} + size_t len() override { return map.size(); } + iterator iter() override { return make_value_iterator(map.begin(), map.end()); } + Map ↦ +}; + +template +struct ItemsViewImpl : public ItemsView { + explicit ItemsViewImpl(Map &map) : map(map) {} + size_t len() override { return map.size(); } + iterator iter() override { return make_iterator(map.begin(), map.end()); } Map ↦ }; @@ -657,9 +700,11 @@ template , typename... class_ bind_map(handle scope, const std::string &name, Args &&...args) { using KeyType = typename Map::key_type; using MappedType = typename Map::mapped_type; - using KeysView = detail::keys_view; - using ValuesView = detail::values_view; - using ItemsView = detail::items_view; + using StrippedKeyType = detail::remove_cvref_t; + using StrippedMappedType = detail::remove_cvref_t; + using KeysView = detail::keys_view; + using ValuesView = detail::values_view; + using ItemsView = detail::items_view; using Class_ = class_; // If either type is a non-module-local bound type then make the map binding non-local as well; @@ -673,12 +718,57 @@ class_ bind_map(handle scope, const std::string &name, Args && } Class_ cl(scope, name.c_str(), pybind11::module_local(local), std::forward(args)...); - class_ keys_view( - scope, ("KeysView[" + name + "]").c_str(), pybind11::module_local(local)); - class_ values_view( - scope, ("ValuesView[" + name + "]").c_str(), pybind11::module_local(local)); - class_ items_view( - scope, ("ItemsView[" + name + "]").c_str(), pybind11::module_local(local)); + static constexpr auto key_type_descr = detail::make_caster::name; + static constexpr auto mapped_type_descr = detail::make_caster::name; + std::string key_type_name(key_type_descr.text), mapped_type_name(mapped_type_descr.text); + + // If key type isn't properly wrapped, fall back to C++ names + if (key_type_name == "%") { + key_type_name = detail::type_info_description(typeid(KeyType)); + } + // Similarly for value type: + if (mapped_type_name == "%") { + mapped_type_name = detail::type_info_description(typeid(MappedType)); + } + + // Wrap KeysView[KeyType] if it wasn't already wrapped + if (!detail::get_type_info(typeid(KeysView))) { + class_ keys_view( + scope, ("KeysView[" + key_type_name + "]").c_str(), pybind11::module_local(local)); + keys_view.def("__len__", &KeysView::len); + keys_view.def("__iter__", + &KeysView::iter, + keep_alive<0, 1>() /* Essential: keep view alive while iterator exists */ + ); + keys_view.def("__contains__", + static_cast(&KeysView::contains)); + // Fallback for when the object is not of the key type + keys_view.def("__contains__", + static_cast(&KeysView::contains)); + } + // Similarly for ValuesView: + if (!detail::get_type_info(typeid(ValuesView))) { + class_ values_view(scope, + ("ValuesView[" + mapped_type_name + "]").c_str(), + pybind11::module_local(local)); + values_view.def("__len__", &ValuesView::len); + values_view.def("__iter__", + &ValuesView::iter, + keep_alive<0, 1>() /* Essential: keep view alive while iterator exists */ + ); + } + // Similarly for ItemsView: + if (!detail::get_type_info(typeid(ItemsView))) { + class_ items_view( + scope, + ("ItemsView[" + key_type_name + ", ").append(mapped_type_name + "]").c_str(), + pybind11::module_local(local)); + items_view.def("__len__", &ItemsView::len); + items_view.def("__iter__", + &ItemsView::iter, + keep_alive<0, 1>() /* Essential: keep view alive while iterator exists */ + ); + } cl.def(init<>()); @@ -698,19 +788,25 @@ class_ bind_map(handle scope, const std::string &name, Args && cl.def( "keys", - [](Map &m) { return KeysView{m}; }, + [](Map &m) { + return std::unique_ptr(new detail::KeysViewImpl(m)); + }, keep_alive<0, 1>() /* Essential: keep map alive while view exists */ ); cl.def( "values", - [](Map &m) { return ValuesView{m}; }, + [](Map &m) { + return std::unique_ptr(new detail::ValuesViewImpl(m)); + }, keep_alive<0, 1>() /* Essential: keep map alive while view exists */ ); cl.def( "items", - [](Map &m) { return ItemsView{m}; }, + [](Map &m) { + return std::unique_ptr(new detail::ItemsViewImpl(m)); + }, keep_alive<0, 1>() /* Essential: keep map alive while view exists */ ); @@ -749,36 +845,6 @@ class_ bind_map(handle scope, const std::string &name, Args && cl.def("__len__", &Map::size); - keys_view.def("__len__", [](KeysView &view) { return view.map.size(); }); - keys_view.def( - "__iter__", - [](KeysView &view) { return make_key_iterator(view.map.begin(), view.map.end()); }, - keep_alive<0, 1>() /* Essential: keep view alive while iterator exists */ - ); - keys_view.def("__contains__", [](KeysView &view, const KeyType &k) -> bool { - auto it = view.map.find(k); - if (it == view.map.end()) { - return false; - } - return true; - }); - // Fallback for when the object is not of the key type - keys_view.def("__contains__", [](KeysView &, const object &) -> bool { return false; }); - - values_view.def("__len__", [](ValuesView &view) { return view.map.size(); }); - values_view.def( - "__iter__", - [](ValuesView &view) { return make_value_iterator(view.map.begin(), view.map.end()); }, - keep_alive<0, 1>() /* Essential: keep view alive while iterator exists */ - ); - - items_view.def("__len__", [](ItemsView &view) { return view.map.size(); }); - items_view.def( - "__iter__", - [](ItemsView &view) { return make_iterator(view.map.begin(), view.map.end()); }, - keep_alive<0, 1>() /* Essential: keep view alive while iterator exists */ - ); - return cl; } diff --git a/pybind11/include/pybind11/type_caster_pyobject_ptr.h b/pybind11/include/pybind11/type_caster_pyobject_ptr.h new file mode 100644 index 0000000..aa914f9 --- /dev/null +++ b/pybind11/include/pybind11/type_caster_pyobject_ptr.h @@ -0,0 +1,61 @@ +// Copyright (c) 2023 The pybind Community. + +#pragma once + +#include "detail/common.h" +#include "detail/descr.h" +#include "cast.h" +#include "pytypes.h" + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(detail) + +template <> +class type_caster { +public: + static constexpr auto name = const_name("object"); // See discussion under PR #4601. + + // This overload is purely to guard against accidents. + template ::value, int> = 0> + static handle cast(T &&, return_value_policy, handle /*parent*/) { + static_assert(is_same_ignoring_cvref::value, + "Invalid C++ type T for to-Python conversion (type_caster)."); + return nullptr; // Unreachable. + } + + static handle cast(PyObject *src, return_value_policy policy, handle /*parent*/) { + if (src == nullptr) { + throw error_already_set(); + } + if (PyErr_Occurred()) { + raise_from(PyExc_SystemError, "src != nullptr but PyErr_Occurred()"); + throw error_already_set(); + } + if (policy == return_value_policy::take_ownership) { + return src; + } + if (policy == return_value_policy::reference + || policy == return_value_policy::automatic_reference) { + return handle(src).inc_ref(); + } + pybind11_fail("type_caster::cast(): unsupported return_value_policy: " + + std::to_string(static_cast(policy))); + } + + bool load(handle src, bool) { + value = reinterpret_borrow(src); + return true; + } + + template + using cast_op_type = PyObject *; + + explicit operator PyObject *() { return value.ptr(); } + +private: + object value; +}; + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/pybind11/noxfile.py b/pybind11/noxfile.py index fe36d70..021ced2 100644 --- a/pybind11/noxfile.py +++ b/pybind11/noxfile.py @@ -5,7 +5,17 @@ nox.needs_version = ">=2022.1.7" nox.options.sessions = ["lint", "tests", "tests_packaging"] -PYTHON_VERISONS = ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "pypy3.7", "pypy3.8"] +PYTHON_VERSIONS = [ + "3.6", + "3.7", + "3.8", + "3.9", + "3.10", + "3.11", + "pypy3.7", + "pypy3.8", + "pypy3.9", +] if os.environ.get("CI", None): nox.options.error_on_missing_interpreters = True @@ -17,10 +27,10 @@ def lint(session: nox.Session) -> None: Lint the codebase (except for clang-format/tidy). """ session.install("pre-commit") - session.run("pre-commit", "run", "-a") + session.run("pre-commit", "run", "-a", *session.posargs) -@nox.session(python=PYTHON_VERISONS) +@nox.session(python=PYTHON_VERSIONS) def tests(session: nox.Session) -> None: """ Run the tests (requires a compiler). @@ -48,7 +58,7 @@ def tests_packaging(session: nox.Session) -> None: """ session.install("-r", "tests/requirements.txt", "--prefer-binary") - session.run("pytest", "tests/extra_python_package") + session.run("pytest", "tests/extra_python_package", *session.posargs) @nox.session(reuse_venv=True) diff --git a/pybind11/pybind11/__init__.py b/pybind11/pybind11/__init__.py index a1be969..7c10b30 100644 --- a/pybind11/pybind11/__init__.py +++ b/pybind11/pybind11/__init__.py @@ -1,16 +1,17 @@ import sys -if sys.version_info < (3, 6): +if sys.version_info < (3, 6): # noqa: UP036 msg = "pybind11 does not support Python < 3.6. 2.9 was the last release supporting Python 2.7 and 3.5." raise ImportError(msg) from ._version import __version__, version_info -from .commands import get_cmake_dir, get_include +from .commands import get_cmake_dir, get_include, get_pkgconfig_dir __all__ = ( "version_info", "__version__", "get_include", "get_cmake_dir", + "get_pkgconfig_dir", ) diff --git a/pybind11/pybind11/__main__.py b/pybind11/pybind11/__main__.py index 22fc447..180665c 100644 --- a/pybind11/pybind11/__main__.py +++ b/pybind11/pybind11/__main__.py @@ -4,7 +4,8 @@ import sys import sysconfig -from .commands import get_cmake_dir, get_include +from ._version import __version__ +from .commands import get_cmake_dir, get_include, get_pkgconfig_dir def print_includes() -> None: @@ -24,8 +25,13 @@ def print_includes() -> None: def main() -> None: - parser = argparse.ArgumentParser() + parser.add_argument( + "--version", + action="version", + version=__version__, + help="Print the version and exit.", + ) parser.add_argument( "--includes", action="store_true", @@ -36,6 +42,11 @@ def main() -> None: action="store_true", help="Print the CMake module directory, ideal for setting -Dpybind11_ROOT in CMake.", ) + parser.add_argument( + "--pkgconfigdir", + action="store_true", + help="Print the pkgconfig directory, ideal for setting $PKG_CONFIG_PATH.", + ) args = parser.parse_args() if not sys.argv[1:]: parser.print_help() @@ -43,6 +54,8 @@ def main() -> None: print_includes() if args.cmakedir: print(get_cmake_dir()) + if args.pkgconfigdir: + print(get_pkgconfig_dir()) if __name__ == "__main__": diff --git a/pybind11/pybind11/_version.py b/pybind11/pybind11/_version.py index b78d596..9280fa0 100644 --- a/pybind11/pybind11/_version.py +++ b/pybind11/pybind11/_version.py @@ -8,5 +8,5 @@ def _to_int(s: str) -> Union[int, str]: return s -__version__ = "2.10.0" +__version__ = "2.11.1" version_info = tuple(_to_int(s) for s in __version__.split(".")) diff --git a/pybind11/pybind11/commands.py b/pybind11/pybind11/commands.py index a29c8ca..b11690f 100644 --- a/pybind11/pybind11/commands.py +++ b/pybind11/pybind11/commands.py @@ -3,7 +3,7 @@ DIR = os.path.abspath(os.path.dirname(__file__)) -def get_include(user: bool = False) -> str: # pylint: disable=unused-argument +def get_include(user: bool = False) -> str: # noqa: ARG001 """ Return the path to the pybind11 include directory. The historical "user" argument is unused, and may be removed. @@ -23,3 +23,15 @@ def get_cmake_dir() -> str: msg = "pybind11 not installed, installation required to access the CMake files" raise ImportError(msg) + + +def get_pkgconfig_dir() -> str: + """ + Return the path to the pybind11 pkgconfig directory. + """ + pkgconfig_installed_path = os.path.join(DIR, "share", "pkgconfig") + if os.path.exists(pkgconfig_installed_path): + return pkgconfig_installed_path + + msg = "pybind11 not installed, installation required to access the pkgconfig files" + raise ImportError(msg) diff --git a/pybind11/pybind11/setup_helpers.py b/pybind11/pybind11/setup_helpers.py index 1fd04b9..aeeee9d 100644 --- a/pybind11/pybind11/setup_helpers.py +++ b/pybind11/pybind11/setup_helpers.py @@ -66,8 +66,8 @@ from setuptools import Extension as _Extension from setuptools.command.build_ext import build_ext as _build_ext except ImportError: - from distutils.command.build_ext import build_ext as _build_ext - from distutils.extension import Extension as _Extension + from distutils.command.build_ext import build_ext as _build_ext # type: ignore[assignment] + from distutils.extension import Extension as _Extension # type: ignore[assignment] import distutils.ccompiler import distutils.errors @@ -84,7 +84,7 @@ # directory into your path if it sits beside your setup.py. -class Pybind11Extension(_Extension): # type: ignore[misc] +class Pybind11Extension(_Extension): """ Build a C++11+ Extension module with pybind11. This automatically adds the recommended flags when you init the extension and assumes C++ sources - you @@ -118,7 +118,6 @@ def _add_ldflags(self, flags: List[str]) -> None: self.extra_link_args[:0] = flags def __init__(self, *args: Any, **kwargs: Any) -> None: - self._cxx_level = 0 cxx_std = kwargs.pop("cxx_std", 0) @@ -145,7 +144,6 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: self.cxx_std = cxx_std cflags = [] - ldflags = [] if WIN: cflags += ["/EHsc", "/bigobj"] else: @@ -155,11 +153,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: c_cpp_flags = shlex.split(env_cflags) + shlex.split(env_cppflags) if not any(opt.startswith("-g") for opt in c_cpp_flags): cflags += ["-g0"] - if MACOS: - cflags += ["-stdlib=libc++"] - ldflags += ["-stdlib=libc++"] self._add_cflags(cflags) - self._add_ldflags(ldflags) @property def cxx_std(self) -> int: @@ -174,9 +168,10 @@ def cxx_std(self) -> int: @cxx_std.setter def cxx_std(self, level: int) -> None: - if self._cxx_level: - warnings.warn("You cannot safely change the cxx_level after setting it!") + warnings.warn( + "You cannot safely change the cxx_level after setting it!", stacklevel=2 + ) # MSVC 2015 Update 3 and later only have 14 (and later 17) modes, so # force a valid flag here. @@ -271,7 +266,7 @@ def auto_cpp_level(compiler: Any) -> Union[str, int]: raise RuntimeError(msg) -class build_ext(_build_ext): # type: ignore[misc] # noqa: N801 +class build_ext(_build_ext): # noqa: N801 """ Customized build_ext that allows an auto-search for the highest supported C++ level for Pybind11Extension. This is only needed for the auto-search @@ -341,7 +336,7 @@ def naive_recompile(obj: str, src: str) -> bool: return os.stat(obj).st_mtime < os.stat(src).st_mtime -def no_recompile(obg: str, src: str) -> bool: # pylint: disable=unused-argument +def no_recompile(obg: str, src: str) -> bool: # noqa: ARG001 """ This is the safest but slowest choice (and is the default) - will always recompile sources. @@ -439,7 +434,6 @@ def compile_function( extra_postargs: Optional[List[str]] = None, depends: Optional[List[str]] = None, ) -> Any: - # These lines are directly from distutils.ccompiler.CCompiler macros, objects, extra_postargs, pp_opts, build = compiler._setup_compile( # type: ignore[attr-defined] output_dir, macros, include_dirs, sources, depends, extra_postargs diff --git a/pybind11/pyproject.toml b/pybind11/pyproject.toml index 3ba1b4b..59c15ea 100644 --- a/pybind11/pyproject.toml +++ b/pybind11/pyproject.toml @@ -2,6 +2,7 @@ requires = ["setuptools>=42", "cmake>=3.18", "ninja"] build-backend = "setuptools.build_meta" + [tool.check-manifest] ignore = [ "tests/**", @@ -15,11 +16,6 @@ ignore = [ "noxfile.py", ] -[tool.isort] -# Needs the compiled .so modules and env.py from tests -known_first_party = "env,pybind11_cross_module_tests,pybind11_tests," -# For black compatibility -profile = "black" [tool.mypy] files = ["pybind11"] @@ -30,7 +26,7 @@ enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"] warn_unreachable = true [[tool.mypy.overrides]] -module = ["ghapi.*", "setuptools.*"] +module = ["ghapi.*"] ignore_missing_imports = true @@ -58,4 +54,45 @@ messages_control.disable = [ "invalid-name", "protected-access", "missing-module-docstring", + "unused-argument", # covered by Ruff ARG +] + + +[tool.ruff] +select = [ + "E", "F", "W", # flake8 + "B", # flake8-bugbear + "I", # isort + "N", # pep8-naming + "ARG", # flake8-unused-arguments + "C4", # flake8-comprehensions + "EM", # flake8-errmsg + "ICN", # flake8-import-conventions + "ISC", # flake8-implicit-str-concat + "PGH", # pygrep-hooks + "PIE", # flake8-pie + "PL", # pylint + "PT", # flake8-pytest-style + "RET", # flake8-return + "RUF100", # Ruff-specific + "SIM", # flake8-simplify + "UP", # pyupgrade + "YTT", # flake8-2020 ] +ignore = [ + "PLR", # Design related pylint + "E501", # Line too long (Black is enough) + "PT011", # Too broad with raises in pytest + "PT004", # Fixture that doesn't return needs underscore (no, it is fine) + "SIM118", # iter(x) is not always the same as iter(x.keys()) +] +target-version = "py37" +src = ["src"] +unfixable = ["T20"] +exclude = [] +line-length = 120 +isort.known-first-party = ["env", "pybind11_cross_module_tests", "pybind11_tests"] + +[tool.ruff.per-file-ignores] +"tests/**" = ["EM", "N"] +"tests/test_call_policies.py" = ["PLC1901"] diff --git a/pybind11/setup.cfg b/pybind11/setup.cfg index 36fbfed..92e6c95 100644 --- a/pybind11/setup.cfg +++ b/pybind11/setup.cfg @@ -20,6 +20,7 @@ classifiers = Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 License :: OSI Approved :: BSD License Programming Language :: Python :: Implementation :: PyPy Programming Language :: Python :: Implementation :: CPython @@ -40,11 +41,3 @@ project_urls = [options] python_requires = >=3.6 zip_safe = False - - -[flake8] -max-line-length = 120 -show_source = True -exclude = .git, __pycache__, build, dist, docs, tools, venv -extend-ignore = E203, E722, B950 -extend-select = B9 diff --git a/pybind11/setup.py b/pybind11/setup.py index a149755..9fea7d3 100644 --- a/pybind11/setup.py +++ b/pybind11/setup.py @@ -96,7 +96,7 @@ def get_and_replace( # Use our input files instead when making the SDist (and anything that depends # on it, like a wheel) -class SDist(setuptools.command.sdist.sdist): # type: ignore[misc] +class SDist(setuptools.command.sdist.sdist): def make_release_tree(self, base_dir: str, files: List[str]) -> None: super().make_release_tree(base_dir, files) @@ -127,6 +127,7 @@ def remove_output(*sources: str) -> Iterator[None]: "-DCMAKE_INSTALL_PREFIX=pybind11", "-DBUILD_TESTING=OFF", "-DPYBIND11_NOPYTHON=ON", + "-Dprefix_for_pc_file=${pcfiledir}/../../", ] if "CMAKE_ARGS" in os.environ: fcommand = [ diff --git a/pybind11/tests/CMakeLists.txt b/pybind11/tests/CMakeLists.txt index 7296cd1..80ee9c1 100644 --- a/pybind11/tests/CMakeLists.txt +++ b/pybind11/tests/CMakeLists.txt @@ -5,20 +5,17 @@ # All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) -# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.21) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.21) + cmake_policy(VERSION 3.26) endif() -# Only needed for CMake < 3.5 support -include(CMakeParseArguments) - # Filter out items; print an optional message if any items filtered. This ignores extensions. # # Usage: @@ -128,7 +125,8 @@ set(PYBIND11_TEST_FILES test_custom_type_casters test_custom_type_setup test_docstring_options - test_eigen + test_eigen_matrix + test_eigen_tensor test_enum test_eval test_exceptions @@ -153,7 +151,11 @@ set(PYBIND11_TEST_FILES test_stl_binders test_tagbased_polymorphic test_thread + test_type_caster_pyobject_ptr test_union + test_unnamed_namespace_a + test_unnamed_namespace_b + test_vector_unique_ptr_member test_virtual_functions) # Invoking cmake with something like: @@ -168,7 +170,7 @@ if(PYBIND11_TEST_OVERRIDE) # This allows the override to be done with extensions, preserving backwards compatibility. foreach(test_name ${TEST_FILES_NO_EXT}) if(NOT ${test_name} IN_LIST TEST_OVERRIDE_NO_EXT - )# If not in the whitelist, add to be filtered out. + )# If not in the allowlist, add to be filtered out. list(APPEND PYBIND11_TEST_FILTER ${test_name}) endif() endforeach() @@ -233,7 +235,10 @@ list(GET PYBIND11_EIGEN_VERSION_AND_HASH 1 PYBIND11_EIGEN_VERSION_HASH) # Check if Eigen is available; if not, remove from PYBIND11_TEST_FILES (but # keep it in PYBIND11_PYTEST_FILES, so that we get the "eigen is not installed" # skip message). -list(FIND PYBIND11_TEST_FILES test_eigen.cpp PYBIND11_TEST_FILES_EIGEN_I) +list(FIND PYBIND11_TEST_FILES test_eigen_matrix.cpp PYBIND11_TEST_FILES_EIGEN_I) +if(PYBIND11_TEST_FILES_EIGEN_I EQUAL -1) + list(FIND PYBIND11_TEST_FILES test_eigen_tensor.cpp PYBIND11_TEST_FILES_EIGEN_I) +endif() if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) # Try loading via newer Eigen's Eigen3Config first (bypassing tools/FindEigen3.cmake). # Eigen 3.3.1+ exports a cmake 3.0+ target for handling dependency requirements, but also @@ -288,13 +293,34 @@ if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) set(EIGEN3_VERSION ${EIGEN3_VERSION_STRING}) endif() message(STATUS "Building tests with Eigen v${EIGEN3_VERSION}") + + if(NOT (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.0)) + tests_extra_targets("test_eigen_tensor.py" "eigen_tensor_avoid_stl_array") + endif() + else() - list(REMOVE_AT PYBIND11_TEST_FILES ${PYBIND11_TEST_FILES_EIGEN_I}) + list(FIND PYBIND11_TEST_FILES test_eigen_matrix.cpp PYBIND11_TEST_FILES_EIGEN_I) + if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) + list(REMOVE_AT PYBIND11_TEST_FILES ${PYBIND11_TEST_FILES_EIGEN_I}) + endif() + + list(FIND PYBIND11_TEST_FILES test_eigen_tensor.cpp PYBIND11_TEST_FILES_EIGEN_I) + if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) + list(REMOVE_AT PYBIND11_TEST_FILES ${PYBIND11_TEST_FILES_EIGEN_I}) + endif() message( STATUS "Building tests WITHOUT Eigen, use -DDOWNLOAD_EIGEN=ON on CMake 3.11+ to download") endif() endif() +# Some code doesn't support gcc 4 +if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.0) + list(FIND PYBIND11_TEST_FILES test_eigen_tensor.cpp PYBIND11_TEST_FILES_EIGEN_I) + if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1) + list(REMOVE_AT PYBIND11_TEST_FILES ${PYBIND11_TEST_FILES_EIGEN_I}) + endif() +endif() + # Optional dependency for some tests (boost::variant is only supported with version >= 1.56) find_package(Boost 1.56) diff --git a/pybind11/tests/conftest.py b/pybind11/tests/conftest.py index 02ce263..ad5b47b 100644 --- a/pybind11/tests/conftest.py +++ b/pybind11/tests/conftest.py @@ -7,13 +7,36 @@ import contextlib import difflib import gc +import multiprocessing import re +import sys import textwrap +import traceback import pytest # Early diagnostic for failed imports -import pybind11_tests +try: + import pybind11_tests +except Exception: + # pytest does not show the traceback without this. + traceback.print_exc() + raise + + +@pytest.fixture(scope="session", autouse=True) +def use_multiprocessing_forkserver_on_linux(): + if sys.platform != "linux": + # The default on Windows and macOS is "spawn": If it's not broken, don't fix it. + return + + # Full background: https://github.com/pybind/pybind11/issues/4105#issuecomment-1301004592 + # In a nutshell: fork() after starting threads == flakiness in the form of deadlocks. + # It is actually a well-known pitfall, unfortunately without guard rails. + # "forkserver" is more performant than "spawn" (~9s vs ~13s for tests/test_gil_scoped.py, + # visit the issuecomment link above for details). + multiprocessing.set_start_method("forkserver") + _long_marker = re.compile(r"([0-9])L") _hexadecimal = re.compile(r"0x[0-9a-fA-F]+") @@ -59,9 +82,8 @@ def __eq__(self, other): b = _strip_and_dedent(other).splitlines() if a == b: return True - else: - self.explanation = _make_explanation(a, b) - return False + self.explanation = _make_explanation(a, b) + return False class Unordered(Output): @@ -72,9 +94,8 @@ def __eq__(self, other): b = _split_and_sort(other) if a == b: return True - else: - self.explanation = _make_explanation(a, b) - return False + self.explanation = _make_explanation(a, b) + return False class Capture: @@ -95,9 +116,8 @@ def __eq__(self, other): b = other if a == b: return True - else: - self.explanation = a.explanation - return False + self.explanation = a.explanation + return False def __str__(self): return self.out @@ -114,7 +134,7 @@ def stderr(self): return Output(self.err) -@pytest.fixture +@pytest.fixture() def capture(capsys): """Extended `capsys` with context manager and custom equality operators""" return Capture(capsys) @@ -135,25 +155,22 @@ def __eq__(self, other): b = _strip_and_dedent(other) if a == b: return True - else: - self.explanation = _make_explanation(a.splitlines(), b.splitlines()) - return False + self.explanation = _make_explanation(a.splitlines(), b.splitlines()) + return False def _sanitize_general(s): s = s.strip() s = s.replace("pybind11_tests.", "m.") - s = _long_marker.sub(r"\1", s) - return s + return _long_marker.sub(r"\1", s) def _sanitize_docstring(thing): s = thing.__doc__ - s = _sanitize_general(s) - return s + return _sanitize_general(s) -@pytest.fixture +@pytest.fixture() def doc(): """Sanitize docstrings and add custom failure explanation""" return SanitizedString(_sanitize_docstring) @@ -162,30 +179,20 @@ def doc(): def _sanitize_message(thing): s = str(thing) s = _sanitize_general(s) - s = _hexadecimal.sub("0", s) - return s + return _hexadecimal.sub("0", s) -@pytest.fixture +@pytest.fixture() def msg(): """Sanitize messages and add custom failure explanation""" return SanitizedString(_sanitize_message) -# noinspection PyUnusedLocal -def pytest_assertrepr_compare(op, left, right): +def pytest_assertrepr_compare(op, left, right): # noqa: ARG001 """Hook to insert custom failure explanation""" if hasattr(left, "explanation"): return left.explanation - - -@contextlib.contextmanager -def suppress(exception): - """Suppress the desired exception""" - try: - yield - except exception: - pass + return None def gc_collect(): @@ -196,7 +203,7 @@ def gc_collect(): def pytest_configure(): - pytest.suppress = suppress + pytest.suppress = contextlib.suppress pytest.gc_collect = gc_collect @@ -210,4 +217,5 @@ def pytest_report_header(config): f" {pybind11_tests.compiler_info}" f" {pybind11_tests.cpp_std}" f" {pybind11_tests.PYBIND11_INTERNALS_ID}" + f" PYBIND11_SIMPLE_GIL_MANAGEMENT={pybind11_tests.PYBIND11_SIMPLE_GIL_MANAGEMENT}" ) diff --git a/pybind11/tests/constructor_stats.h b/pybind11/tests/constructor_stats.h index a3835c2..937f6c2 100644 --- a/pybind11/tests/constructor_stats.h +++ b/pybind11/tests/constructor_stats.h @@ -115,7 +115,7 @@ class ConstructorStats { #if defined(PYPY_VERSION) PyObject *globals = PyEval_GetGlobals(); PyObject *result = PyRun_String("import gc\n" - "for i in range(2):" + "for i in range(2):\n" " gc.collect()\n", Py_file_input, globals, diff --git a/pybind11/tests/cross_module_gil_utils.cpp b/pybind11/tests/cross_module_gil_utils.cpp index 1436c35..7c20849 100644 --- a/pybind11/tests/cross_module_gil_utils.cpp +++ b/pybind11/tests/cross_module_gil_utils.cpp @@ -6,9 +6,15 @@ All rights reserved. Use of this source code is governed by a BSD-style license that can be found in the LICENSE file. */ +#if defined(PYBIND11_INTERNALS_VERSION) +# undef PYBIND11_INTERNALS_VERSION +#endif +#define PYBIND11_INTERNALS_VERSION 21814642 // Ensure this module has its own `internals` instance. #include #include +#include +#include // This file mimics a DSO that makes pybind11 calls but does not define a // PYBIND11_MODULE. The purpose is to test that such a DSO can create a @@ -21,8 +27,54 @@ namespace { namespace py = pybind11; + void gil_acquire() { py::gil_scoped_acquire gil; } +std::string gil_multi_acquire_release(unsigned bits) { + if ((bits & 0x1u) != 0u) { + py::gil_scoped_acquire gil; + } + if ((bits & 0x2u) != 0u) { + py::gil_scoped_release gil; + } + if ((bits & 0x4u) != 0u) { + py::gil_scoped_acquire gil; + } + if ((bits & 0x8u) != 0u) { + py::gil_scoped_release gil; + } + return PYBIND11_INTERNALS_ID; +} + +struct CustomAutoGIL { + CustomAutoGIL() : gstate(PyGILState_Ensure()) {} + ~CustomAutoGIL() { PyGILState_Release(gstate); } + + PyGILState_STATE gstate; +}; +struct CustomAutoNoGIL { + CustomAutoNoGIL() : save(PyEval_SaveThread()) {} + ~CustomAutoNoGIL() { PyEval_RestoreThread(save); } + + PyThreadState *save; +}; + +template +void gil_acquire_inner() { + Acquire acquire_outer; + Acquire acquire_inner; + Release release; +} + +template +void gil_acquire_nested() { + Acquire acquire_outer; + Acquire acquire_inner; + Release release; + auto thread = std::thread(&gil_acquire_inner); + thread.join(); +} + constexpr char kModuleName[] = "cross_module_gil_utils"; struct PyModuleDef moduledef = { @@ -30,6 +82,9 @@ struct PyModuleDef moduledef = { } // namespace +#define ADD_FUNCTION(Name, ...) \ + PyModule_AddObject(m, Name, PyLong_FromVoidPtr(reinterpret_cast(&__VA_ARGS__))); + extern "C" PYBIND11_EXPORT PyObject *PyInit_cross_module_gil_utils() { PyObject *m = PyModule_Create(&moduledef); @@ -37,8 +92,16 @@ extern "C" PYBIND11_EXPORT PyObject *PyInit_cross_module_gil_utils() { if (m != nullptr) { static_assert(sizeof(&gil_acquire) == sizeof(void *), "Function pointer must have the same size as void*"); - PyModule_AddObject( - m, "gil_acquire_funcaddr", PyLong_FromVoidPtr(reinterpret_cast(&gil_acquire))); + ADD_FUNCTION("gil_acquire_funcaddr", gil_acquire) + ADD_FUNCTION("gil_multi_acquire_release_funcaddr", gil_multi_acquire_release) + ADD_FUNCTION("gil_acquire_inner_custom_funcaddr", + gil_acquire_inner) + ADD_FUNCTION("gil_acquire_nested_custom_funcaddr", + gil_acquire_nested) + ADD_FUNCTION("gil_acquire_inner_pybind11_funcaddr", + gil_acquire_inner) + ADD_FUNCTION("gil_acquire_nested_pybind11_funcaddr", + gil_acquire_nested) } return m; diff --git a/pybind11/tests/eigen_tensor_avoid_stl_array.cpp b/pybind11/tests/eigen_tensor_avoid_stl_array.cpp new file mode 100644 index 0000000..eacc9e9 --- /dev/null +++ b/pybind11/tests/eigen_tensor_avoid_stl_array.cpp @@ -0,0 +1,14 @@ +/* + tests/eigen_tensor.cpp -- automatic conversion of Eigen Tensor + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#ifndef EIGEN_AVOID_STL_ARRAY +# define EIGEN_AVOID_STL_ARRAY +#endif + +#include "test_eigen_tensor.inl" + +PYBIND11_MODULE(eigen_tensor_avoid_stl_array, m) { eigen_tensor_test::test_module(m); } diff --git a/pybind11/tests/env.py b/pybind11/tests/env.py index 0345df6..7eea5a3 100644 --- a/pybind11/tests/env.py +++ b/pybind11/tests/env.py @@ -24,5 +24,4 @@ def deprecated_call(): pytest_major_minor = (int(pieces[0]), int(pieces[1])) if pytest_major_minor < (3, 9): return pytest.warns((DeprecationWarning, PendingDeprecationWarning)) - else: - return pytest.deprecated_call() + return pytest.deprecated_call() diff --git a/pybind11/tests/extra_python_package/test_files.py b/pybind11/tests/extra_python_package/test_files.py index ba16b52..57387dd 100644 --- a/pybind11/tests/extra_python_package/test_files.py +++ b/pybind11/tests/extra_python_package/test_files.py @@ -12,6 +12,16 @@ DIR = os.path.abspath(os.path.dirname(__file__)) MAIN_DIR = os.path.dirname(os.path.dirname(DIR)) +PKGCONFIG = """\ +prefix=${{pcfiledir}}/../../ +includedir=${{prefix}}/include + +Name: pybind11 +Description: Seamless operability between C++11 and Python +Version: {VERSION} +Cflags: -I${{includedir}} +""" + main_headers = { "include/pybind11/attr.h", @@ -33,6 +43,7 @@ "include/pybind11/pytypes.h", "include/pybind11/stl.h", "include/pybind11/stl_bind.h", + "include/pybind11/type_caster_pyobject_ptr.h", } detail_headers = { @@ -45,6 +56,12 @@ "include/pybind11/detail/typeid.h", } +eigen_headers = { + "include/pybind11/eigen/common.h", + "include/pybind11/eigen/matrix.h", + "include/pybind11/eigen/tensor.h", +} + stl_headers = { "include/pybind11/stl/filesystem.h", } @@ -59,6 +76,10 @@ "share/cmake/pybind11/pybind11Tools.cmake", } +pkgconfig_files = { + "share/pkgconfig/pybind11.pc", +} + py_files = { "__init__.py", "__main__.py", @@ -68,8 +89,8 @@ "setup_helpers.py", } -headers = main_headers | detail_headers | stl_headers -src_files = headers | cmake_files +headers = main_headers | detail_headers | eigen_headers | stl_headers +src_files = headers | cmake_files | pkgconfig_files all_files = src_files | py_files @@ -78,10 +99,12 @@ "pybind11/include", "pybind11/include/pybind11", "pybind11/include/pybind11/detail", + "pybind11/include/pybind11/eigen", "pybind11/include/pybind11/stl", "pybind11/share", "pybind11/share/cmake", "pybind11/share/cmake/pybind11", + "pybind11/share/pkgconfig", "pyproject.toml", "setup.cfg", "setup.py", @@ -89,6 +112,7 @@ "MANIFEST.in", "README.rst", "PKG-INFO", + "SECURITY.md", } local_sdist_files = { @@ -101,22 +125,24 @@ } -def test_build_sdist(monkeypatch, tmpdir): +def read_tz_file(tar: tarfile.TarFile, name: str) -> bytes: + start = tar.getnames()[0] + "/" + inner_file = tar.extractfile(tar.getmember(f"{start}{name}")) + assert inner_file + with contextlib.closing(inner_file) as f: + return f.read() + + +def normalize_line_endings(value: bytes) -> bytes: + return value.replace(os.linesep.encode("utf-8"), b"\n") + +def test_build_sdist(monkeypatch, tmpdir): monkeypatch.chdir(MAIN_DIR) - out = subprocess.check_output( - [ - sys.executable, - "-m", - "build", - "--sdist", - "--outdir", - str(tmpdir), - ] + subprocess.run( + [sys.executable, "-m", "build", "--sdist", f"--outdir={tmpdir}"], check=True ) - if hasattr(out, "decode"): - out = out.decode() (sdist,) = tmpdir.visit("*.tar.gz") @@ -125,25 +151,17 @@ def test_build_sdist(monkeypatch, tmpdir): version = start[9:-1] simpler = {n.split("/", 1)[-1] for n in tar.getnames()[1:]} - with contextlib.closing( - tar.extractfile(tar.getmember(start + "setup.py")) - ) as f: - setup_py = f.read() - - with contextlib.closing( - tar.extractfile(tar.getmember(start + "pyproject.toml")) - ) as f: - pyproject_toml = f.read() - - with contextlib.closing( - tar.extractfile( - tar.getmember( - start + "pybind11/share/cmake/pybind11/pybind11Config.cmake" - ) - ) - ) as f: - contents = f.read().decode("utf8") - assert 'set(pybind11_INCLUDE_DIR "${PACKAGE_PREFIX_DIR}/include")' in contents + setup_py = read_tz_file(tar, "setup.py") + pyproject_toml = read_tz_file(tar, "pyproject.toml") + pkgconfig = read_tz_file(tar, "pybind11/share/pkgconfig/pybind11.pc") + cmake_cfg = read_tz_file( + tar, "pybind11/share/cmake/pybind11/pybind11Config.cmake" + ) + + assert ( + 'set(pybind11_INCLUDE_DIR "${PACKAGE_PREFIX_DIR}/include")' + in cmake_cfg.decode("utf-8") + ) files = {f"pybind11/{n}" for n in all_files} files |= sdist_files @@ -154,9 +172,9 @@ def test_build_sdist(monkeypatch, tmpdir): with open(os.path.join(MAIN_DIR, "tools", "setup_main.py.in"), "rb") as f: contents = ( - string.Template(f.read().decode()) + string.Template(f.read().decode("utf-8")) .substitute(version=version, extra_cmd="") - .encode() + .encode("utf-8") ) assert setup_py == contents @@ -164,25 +182,18 @@ def test_build_sdist(monkeypatch, tmpdir): contents = f.read() assert pyproject_toml == contents + simple_version = ".".join(version.split(".")[:3]) + pkgconfig_expected = PKGCONFIG.format(VERSION=simple_version).encode("utf-8") + assert normalize_line_endings(pkgconfig) == pkgconfig_expected -def test_build_global_dist(monkeypatch, tmpdir): +def test_build_global_dist(monkeypatch, tmpdir): monkeypatch.chdir(MAIN_DIR) monkeypatch.setenv("PYBIND11_GLOBAL_SDIST", "1") - out = subprocess.check_output( - [ - sys.executable, - "-m", - "build", - "--sdist", - "--outdir", - str(tmpdir), - ] + subprocess.run( + [sys.executable, "-m", "build", "--sdist", "--outdir", str(tmpdir)], check=True ) - if hasattr(out, "decode"): - out = out.decode() - (sdist,) = tmpdir.visit("*.tar.gz") with tarfile.open(str(sdist), "r:gz") as tar: @@ -190,15 +201,17 @@ def test_build_global_dist(monkeypatch, tmpdir): version = start[16:-1] simpler = {n.split("/", 1)[-1] for n in tar.getnames()[1:]} - with contextlib.closing( - tar.extractfile(tar.getmember(start + "setup.py")) - ) as f: - setup_py = f.read() + setup_py = read_tz_file(tar, "setup.py") + pyproject_toml = read_tz_file(tar, "pyproject.toml") + pkgconfig = read_tz_file(tar, "pybind11/share/pkgconfig/pybind11.pc") + cmake_cfg = read_tz_file( + tar, "pybind11/share/cmake/pybind11/pybind11Config.cmake" + ) - with contextlib.closing( - tar.extractfile(tar.getmember(start + "pyproject.toml")) - ) as f: - pyproject_toml = f.read() + assert ( + 'set(pybind11_INCLUDE_DIR "${PACKAGE_PREFIX_DIR}/include")' + in cmake_cfg.decode("utf-8") + ) files = {f"pybind11/{n}" for n in all_files} files |= sdist_files @@ -209,7 +222,7 @@ def test_build_global_dist(monkeypatch, tmpdir): contents = ( string.Template(f.read().decode()) .substitute(version=version, extra_cmd="") - .encode() + .encode("utf-8") ) assert setup_py == contents @@ -217,12 +230,16 @@ def test_build_global_dist(monkeypatch, tmpdir): contents = f.read() assert pyproject_toml == contents + simple_version = ".".join(version.split(".")[:3]) + pkgconfig_expected = PKGCONFIG.format(VERSION=simple_version).encode("utf-8") + assert normalize_line_endings(pkgconfig) == pkgconfig_expected + def tests_build_wheel(monkeypatch, tmpdir): monkeypatch.chdir(MAIN_DIR) - subprocess.check_output( - [sys.executable, "-m", "pip", "wheel", ".", "-w", str(tmpdir)] + subprocess.run( + [sys.executable, "-m", "pip", "wheel", ".", "-w", str(tmpdir)], check=True ) (wheel,) = tmpdir.visit("*.whl") @@ -249,8 +266,8 @@ def tests_build_global_wheel(monkeypatch, tmpdir): monkeypatch.chdir(MAIN_DIR) monkeypatch.setenv("PYBIND11_GLOBAL_SDIST", "1") - subprocess.check_output( - [sys.executable, "-m", "pip", "wheel", ".", "-w", str(tmpdir)] + subprocess.run( + [sys.executable, "-m", "pip", "wheel", ".", "-w", str(tmpdir)], check=True ) (wheel,) = tmpdir.visit("*.whl") diff --git a/pybind11/tests/pybind11_tests.cpp b/pybind11/tests/pybind11_tests.cpp index aa30955..6240346 100644 --- a/pybind11/tests/pybind11_tests.cpp +++ b/pybind11/tests/pybind11_tests.cpp @@ -89,6 +89,12 @@ PYBIND11_MODULE(pybind11_tests, m) { #endif m.attr("cpp_std") = cpp_std(); m.attr("PYBIND11_INTERNALS_ID") = PYBIND11_INTERNALS_ID; + m.attr("PYBIND11_SIMPLE_GIL_MANAGEMENT") = +#if defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) + true; +#else + false; +#endif bind_ConstructorStats(m); diff --git a/pybind11/tests/requirements.txt b/pybind11/tests/requirements.txt index 04aafa8..4ba1011 100644 --- a/pybind11/tests/requirements.txt +++ b/pybind11/tests/requirements.txt @@ -6,4 +6,4 @@ numpy==1.22.2; platform_python_implementation!="PyPy" and python_version>="3.10" pytest==7.0.0 pytest-timeout scipy==1.5.4; platform_python_implementation!="PyPy" and python_version<"3.10" -scipy==1.8.0; platform_python_implementation!="PyPy" and python_version=="3.10" +scipy==1.10.0; platform_python_implementation!="PyPy" and python_version=="3.10" diff --git a/pybind11/tests/test_async.py b/pybind11/tests/test_async.py index b9ff951..83a4c50 100644 --- a/pybind11/tests/test_async.py +++ b/pybind11/tests/test_async.py @@ -4,7 +4,7 @@ m = pytest.importorskip("pybind11_tests.async_module") -@pytest.fixture +@pytest.fixture() def event_loop(): loop = asyncio.new_event_loop() yield loop @@ -16,7 +16,7 @@ async def get_await_result(x): def test_await(event_loop): - assert 5 == event_loop.run_until_complete(get_await_result(m.SupportsAsync())) + assert event_loop.run_until_complete(get_await_result(m.SupportsAsync())) == 5 def test_await_missing(event_loop): diff --git a/pybind11/tests/test_buffers.cpp b/pybind11/tests/test_buffers.cpp index 6b6e8cb..b5b8c35 100644 --- a/pybind11/tests/test_buffers.cpp +++ b/pybind11/tests/test_buffers.cpp @@ -7,12 +7,47 @@ BSD-style license that can be found in the LICENSE file. */ +#include #include #include "constructor_stats.h" #include "pybind11_tests.h" TEST_SUBMODULE(buffers, m) { + m.attr("long_double_and_double_have_same_size") = (sizeof(long double) == sizeof(double)); + + m.def("format_descriptor_format_buffer_info_equiv", + [](const std::string &cpp_name, const py::buffer &buffer) { + // https://google.github.io/styleguide/cppguide.html#Static_and_Global_Variables + static auto *format_table = new std::map; + static auto *equiv_table + = new std::map; + if (format_table->empty()) { +#define PYBIND11_ASSIGN_HELPER(...) \ + (*format_table)[#__VA_ARGS__] = py::format_descriptor<__VA_ARGS__>::format(); \ + (*equiv_table)[#__VA_ARGS__] = &py::buffer_info::item_type_is_equivalent_to<__VA_ARGS__>; + PYBIND11_ASSIGN_HELPER(PyObject *) + PYBIND11_ASSIGN_HELPER(bool) + PYBIND11_ASSIGN_HELPER(std::int8_t) + PYBIND11_ASSIGN_HELPER(std::uint8_t) + PYBIND11_ASSIGN_HELPER(std::int16_t) + PYBIND11_ASSIGN_HELPER(std::uint16_t) + PYBIND11_ASSIGN_HELPER(std::int32_t) + PYBIND11_ASSIGN_HELPER(std::uint32_t) + PYBIND11_ASSIGN_HELPER(std::int64_t) + PYBIND11_ASSIGN_HELPER(std::uint64_t) + PYBIND11_ASSIGN_HELPER(float) + PYBIND11_ASSIGN_HELPER(double) + PYBIND11_ASSIGN_HELPER(long double) + PYBIND11_ASSIGN_HELPER(std::complex) + PYBIND11_ASSIGN_HELPER(std::complex) + PYBIND11_ASSIGN_HELPER(std::complex) +#undef PYBIND11_ASSIGN_HELPER + } + return std::pair( + (*format_table)[cpp_name], (buffer.request().*((*equiv_table)[cpp_name]))()); + }); + // test_from_python / test_to_python: class Matrix { public: diff --git a/pybind11/tests/test_buffers.py b/pybind11/tests/test_buffers.py index 8354b68..63d9d86 100644 --- a/pybind11/tests/test_buffers.py +++ b/pybind11/tests/test_buffers.py @@ -10,6 +10,63 @@ np = pytest.importorskip("numpy") +if m.long_double_and_double_have_same_size: + # Determined by the compiler used to build the pybind11 tests + # (e.g. MSVC gets here, but MinGW might not). + np_float128 = None + np_complex256 = None +else: + # Determined by the compiler used to build numpy (e.g. MinGW). + np_float128 = getattr(np, *["float128"] * 2) + np_complex256 = getattr(np, *["complex256"] * 2) + +CPP_NAME_FORMAT_NP_DTYPE_TABLE = [ + ("PyObject *", "O", object), + ("bool", "?", np.bool_), + ("std::int8_t", "b", np.int8), + ("std::uint8_t", "B", np.uint8), + ("std::int16_t", "h", np.int16), + ("std::uint16_t", "H", np.uint16), + ("std::int32_t", "i", np.int32), + ("std::uint32_t", "I", np.uint32), + ("std::int64_t", "q", np.int64), + ("std::uint64_t", "Q", np.uint64), + ("float", "f", np.float32), + ("double", "d", np.float64), + ("long double", "g", np_float128), + ("std::complex", "Zf", np.complex64), + ("std::complex", "Zd", np.complex128), + ("std::complex", "Zg", np_complex256), +] +CPP_NAME_FORMAT_TABLE = [ + (cpp_name, format) + for cpp_name, format, np_dtype in CPP_NAME_FORMAT_NP_DTYPE_TABLE + if np_dtype is not None +] +CPP_NAME_NP_DTYPE_TABLE = [ + (cpp_name, np_dtype) for cpp_name, _, np_dtype in CPP_NAME_FORMAT_NP_DTYPE_TABLE +] + + +@pytest.mark.parametrize(("cpp_name", "np_dtype"), CPP_NAME_NP_DTYPE_TABLE) +def test_format_descriptor_format_buffer_info_equiv(cpp_name, np_dtype): + if np_dtype is None: + pytest.skip( + f"cpp_name=`{cpp_name}`: `long double` and `double` have same size." + ) + if isinstance(np_dtype, str): + pytest.skip(f"np.{np_dtype} does not exist.") + np_array = np.array([], dtype=np_dtype) + for other_cpp_name, expected_format in CPP_NAME_FORMAT_TABLE: + format, np_array_is_matching = m.format_descriptor_format_buffer_info_equiv( + other_cpp_name, np_array + ) + assert format == expected_format + if other_cpp_name == cpp_name: + assert np_array_is_matching + else: + assert not np_array_is_matching + def test_from_python(): with pytest.raises(RuntimeError) as excinfo: @@ -54,7 +111,8 @@ def test_to_python(): mat2 = np.array(mat, copy=False) assert mat2.shape == (5, 4) assert abs(mat2).sum() == 11 - assert mat2[2, 3] == 4 and mat2[3, 2] == 7 + assert mat2[2, 3] == 4 + assert mat2[3, 2] == 7 mat2[2, 3] = 5 assert mat2[2, 3] == 5 diff --git a/pybind11/tests/test_builtin_casters.cpp b/pybind11/tests/test_builtin_casters.cpp index 6529f47..0623b85 100644 --- a/pybind11/tests/test_builtin_casters.cpp +++ b/pybind11/tests/test_builtin_casters.cpp @@ -73,6 +73,9 @@ PYBIND11_NAMESPACE_END(detail) PYBIND11_NAMESPACE_END(pybind11) TEST_SUBMODULE(builtin_casters, m) { + PYBIND11_WARNING_PUSH + PYBIND11_WARNING_DISABLE_MSVC(4127) + // test_simple_string m.def("string_roundtrip", [](const char *s) { return s; }); @@ -86,7 +89,7 @@ TEST_SUBMODULE(builtin_casters, m) { std::wstring wstr; wstr.push_back(0x61); // a wstr.push_back(0x2e18); // ⸘ - if (PYBIND11_SILENCE_MSVC_C4127(sizeof(wchar_t) == 2)) { + if (sizeof(wchar_t) == 2) { wstr.push_back(mathbfA16_1); wstr.push_back(mathbfA16_2); } // 𝐀, utf16 @@ -113,7 +116,7 @@ TEST_SUBMODULE(builtin_casters, m) { // Under Python 2.7, invalid unicode UTF-32 characters didn't appear to trigger // UnicodeDecodeError m.def("bad_utf32_string", [=]() { return std::u32string({a32, char32_t(0xd800), z32}); }); - if (PYBIND11_SILENCE_MSVC_C4127(sizeof(wchar_t) == 2)) { + if (sizeof(wchar_t) == 2) { m.def("bad_wchar_string", [=]() { return std::wstring({wchar_t(0x61), wchar_t(0xd800)}); }); @@ -266,9 +269,14 @@ TEST_SUBMODULE(builtin_casters, m) { }); m.def("lvalue_nested", []() -> const decltype(lvnested) & { return lvnested; }); - static std::pair int_string_pair{2, "items"}; m.def( - "int_string_pair", []() { return &int_string_pair; }, py::return_value_policy::reference); + "int_string_pair", + []() { + // Using no-destructor idiom to side-step warnings from overzealous compilers. + static auto *int_string_pair = new std::pair{2, "items"}; + return int_string_pair; + }, + py::return_value_policy::reference); // test_builtins_cast_return_none m.def("return_none_string", []() -> std::string * { return nullptr; }); @@ -379,4 +387,6 @@ TEST_SUBMODULE(builtin_casters, m) { m.def("takes_const_ref", [](const ConstRefCasted &x) { return x.tag; }); m.def("takes_const_ref_wrap", [](std::reference_wrapper x) { return x.get().tag; }); + + PYBIND11_WARNING_POP } diff --git a/pybind11/tests/test_builtin_casters.py b/pybind11/tests/test_builtin_casters.py index d38ae68..b1f57bd 100644 --- a/pybind11/tests/test_builtin_casters.py +++ b/pybind11/tests/test_builtin_casters.py @@ -126,8 +126,8 @@ def test_bytes_to_string(): assert m.strlen(b"hi") == 2 assert m.string_length(b"world") == 5 - assert m.string_length("a\x00b".encode()) == 3 - assert m.strlen("a\x00b".encode()) == 1 # C-string limitation + assert m.string_length(b"a\x00b") == 3 + assert m.strlen(b"a\x00b") == 1 # C-string limitation # passing in a utf8 encoded string should work assert m.string_length("πŸ’©".encode()) == 4 @@ -421,13 +421,15 @@ def test_reference_wrapper(): a2 = m.refwrap_list(copy=True) assert [x.value for x in a1] == [2, 3] assert [x.value for x in a2] == [2, 3] - assert not a1[0] is a2[0] and not a1[1] is a2[1] + assert a1[0] is not a2[0] + assert a1[1] is not a2[1] b1 = m.refwrap_list(copy=False) b2 = m.refwrap_list(copy=False) assert [x.value for x in b1] == [1, 2] assert [x.value for x in b2] == [1, 2] - assert b1[0] is b2[0] and b1[1] is b2[1] + assert b1[0] is b2[0] + assert b1[1] is b2[1] assert m.refwrap_iiw(IncType(5)) == 5 assert m.refwrap_call_iiw(IncType(10), m.refwrap_iiw) == [10, 10, 10, 10] diff --git a/pybind11/tests/test_callbacks.cpp b/pybind11/tests/test_callbacks.cpp index 92b8053..2fd05de 100644 --- a/pybind11/tests/test_callbacks.cpp +++ b/pybind11/tests/test_callbacks.cpp @@ -240,4 +240,41 @@ TEST_SUBMODULE(callbacks, m) { f(); } }); + + auto *custom_def = []() { + static PyMethodDef def; + def.ml_name = "example_name"; + def.ml_doc = "Example doc"; + def.ml_meth = [](PyObject *, PyObject *args) -> PyObject * { + if (PyTuple_Size(args) != 1) { + throw std::runtime_error("Invalid number of arguments for example_name"); + } + PyObject *first = PyTuple_GetItem(args, 0); + if (!PyLong_Check(first)) { + throw std::runtime_error("Invalid argument to example_name"); + } + auto result = py::cast(PyLong_AsLong(first) * 9); + return result.release().ptr(); + }; + def.ml_flags = METH_VARARGS; + return &def; + }(); + + // rec_capsule with name that has the same value (but not pointer) as our internal one + // This capsule should be detected by our code as foreign and not inspected as the pointers + // shouldn't match + constexpr const char *rec_capsule_name + = pybind11::detail::internals_function_record_capsule_name; + py::capsule rec_capsule(std::malloc(1), [](void *data) { std::free(data); }); + rec_capsule.set_name(rec_capsule_name); + m.add_object("custom_function", PyCFunction_New(custom_def, rec_capsule.ptr())); + + // This test requires a new ABI version to pass +#if PYBIND11_INTERNALS_VERSION > 4 + // rec_capsule with nullptr name + py::capsule rec_capsule2(std::malloc(1), [](void *data) { std::free(data); }); + m.add_object("custom_function2", PyCFunction_New(custom_def, rec_capsule2.ptr())); +#else + m.add_object("custom_function2", py::none()); +#endif } diff --git a/pybind11/tests/test_callbacks.py b/pybind11/tests/test_callbacks.py index 0b1047b..4a652f5 100644 --- a/pybind11/tests/test_callbacks.py +++ b/pybind11/tests/test_callbacks.py @@ -5,6 +5,7 @@ import env # noqa: F401 from pybind11_tests import callbacks as m +from pybind11_tests import detailed_error_messages_enabled def test_callbacks(): @@ -70,11 +71,20 @@ def f(*args, **kwargs): with pytest.raises(RuntimeError) as excinfo: m.test_arg_conversion_error1(f) - assert "Unable to convert call argument" in str(excinfo.value) + assert str(excinfo.value) == "Unable to convert call argument " + ( + "'1' of type 'UnregisteredType' to Python object" + if detailed_error_messages_enabled + else "'1' to Python object (#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)" + ) with pytest.raises(RuntimeError) as excinfo: m.test_arg_conversion_error2(f) - assert "Unable to convert call argument" in str(excinfo.value) + assert str(excinfo.value) == "Unable to convert call argument " + ( + "'expected_name' of type 'UnregisteredType' to Python object" + if detailed_error_messages_enabled + else "'expected_name' to Python object " + "(#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)" + ) def test_lambda_closure_cleanup(): @@ -193,3 +203,16 @@ def test_callback_num_times(): if len(rates) > 1: print("Min Mean Max") print(f"{min(rates):6.3f} {sum(rates) / len(rates):6.3f} {max(rates):6.3f}") + + +def test_custom_func(): + assert m.custom_function(4) == 36 + assert m.roundtrip(m.custom_function)(4) == 36 + + +@pytest.mark.skipif( + m.custom_function2 is None, reason="Current PYBIND11_INTERNALS_VERSION too low" +) +def test_custom_func2(): + assert m.custom_function2(3) == 27 + assert m.roundtrip(m.custom_function2)(3) == 27 diff --git a/pybind11/tests/test_chrono.py b/pybind11/tests/test_chrono.py index 7f47b37..a29316c 100644 --- a/pybind11/tests/test_chrono.py +++ b/pybind11/tests/test_chrono.py @@ -7,7 +7,6 @@ def test_chrono_system_clock(): - # Get the time from both c++ and datetime date0 = datetime.datetime.today() date1 = m.test_chrono1() @@ -122,7 +121,6 @@ def test_chrono_system_clock_roundtrip_time(time1, tz, monkeypatch): def test_chrono_duration_roundtrip(): - # Get the difference between two times (a timedelta) date1 = datetime.datetime.today() date2 = datetime.datetime.today() @@ -143,7 +141,6 @@ def test_chrono_duration_roundtrip(): def test_chrono_duration_subtraction_equivalence(): - date1 = datetime.datetime.today() date2 = datetime.datetime.today() @@ -154,7 +151,6 @@ def test_chrono_duration_subtraction_equivalence(): def test_chrono_duration_subtraction_equivalence_date(): - date1 = datetime.date.today() date2 = datetime.date.today() diff --git a/pybind11/tests/test_class.cpp b/pybind11/tests/test_class.cpp index c8b8071..7241bc8 100644 --- a/pybind11/tests/test_class.cpp +++ b/pybind11/tests/test_class.cpp @@ -22,10 +22,8 @@ #include -#if defined(_MSC_VER) -# pragma warning(disable : 4324) +PYBIND11_WARNING_DISABLE_MSVC(4324) // warning C4324: structure was padded due to alignment specifier -#endif // test_brace_initialization struct NoBraceInitialization { @@ -36,7 +34,29 @@ struct NoBraceInitialization { std::vector vec; }; +namespace test_class { +namespace pr4220_tripped_over_this { // PR #4227 + +template +struct SoEmpty {}; + +template +std::string get_msg(const T &) { + return "This is really only meant to exercise successful compilation."; +} + +using Empty0 = SoEmpty<0x0>; + +void bind_empty0(py::module_ &m) { + py::class_(m, "Empty0").def(py::init<>()).def("get_msg", get_msg); +} + +} // namespace pr4220_tripped_over_this +} // namespace test_class + TEST_SUBMODULE(class_, m) { + m.def("obj_class_name", [](py::handle obj) { return py::detail::obj_class_name(obj.ptr()); }); + // test_instance struct NoConstructor { NoConstructor() = default; @@ -65,7 +85,7 @@ TEST_SUBMODULE(class_, m) { .def_static("new_instance", &NoConstructor::new_instance, "Return an instance"); py::class_(m, "NoConstructorNew") - .def(py::init([](const NoConstructorNew &self) { return self; })) // Need a NOOP __init__ + .def(py::init([]() { return nullptr; })) // Need a NOOP __init__ .def_static("__new__", [](const py::object &) { return NoConstructorNew::new_instance(); }); @@ -364,6 +384,8 @@ TEST_SUBMODULE(class_, m) { protected: virtual int foo() const { return value; } + virtual void *void_foo() { return static_cast(&value); } + virtual void *get_self() { return static_cast(this); } private: int value = 42; @@ -372,6 +394,8 @@ TEST_SUBMODULE(class_, m) { class TrampolineB : public ProtectedB { public: int foo() const override { PYBIND11_OVERRIDE(int, ProtectedB, foo, ); } + void *void_foo() override { PYBIND11_OVERRIDE(void *, ProtectedB, void_foo, ); } + void *get_self() override { PYBIND11_OVERRIDE(void *, ProtectedB, get_self, ); } }; class PublicistB : public ProtectedB { @@ -381,11 +405,23 @@ TEST_SUBMODULE(class_, m) { // (in Debug builds only, tested with icpc (ICC) 2021.1 Beta 20200827) ~PublicistB() override{}; // NOLINT(modernize-use-equals-default) using ProtectedB::foo; + using ProtectedB::get_self; + using ProtectedB::void_foo; }; + m.def("read_foo", [](const void *original) { + const int *ptr = reinterpret_cast(original); + return *ptr; + }); + + m.def("pointers_equal", + [](const void *original, const void *comparison) { return original == comparison; }); + py::class_(m, "ProtectedB") .def(py::init<>()) - .def("foo", &PublicistB::foo); + .def("foo", &PublicistB::foo) + .def("void_foo", &PublicistB::void_foo) + .def("get_self", &PublicistB::get_self); // test_brace_initialization struct BraceInitialization { @@ -517,6 +553,8 @@ TEST_SUBMODULE(class_, m) { py::class_(gt, "OtherDuplicateNested"); py::class_(gt, "YetAnotherDuplicateNested"); }); + + test_class::pr4220_tripped_over_this::bind_empty0(m); } template diff --git a/pybind11/tests/test_class.py b/pybind11/tests/test_class.py index ff9196f..ee7467c 100644 --- a/pybind11/tests/test_class.py +++ b/pybind11/tests/test_class.py @@ -1,10 +1,16 @@ import pytest -import env # noqa: F401 +import env from pybind11_tests import ConstructorStats, UserType from pybind11_tests import class_ as m +def test_obj_class_name(): + expected_name = "UserType" if env.PYPY else "pybind11_tests.UserType" + assert m.obj_class_name(UserType(1)) == expected_name + assert m.obj_class_name(UserType) == expected_name + + def test_repr(): assert "pybind11_type" in repr(type(UserType)) assert "UserType" in repr(UserType) @@ -23,7 +29,7 @@ def test_instance(msg): assert cstats.alive() == 0 -def test_instance_new(msg): +def test_instance_new(): instance = m.NoConstructorNew() # .__new__(m.NoConstructor.__class__) cstats = ConstructorStats.get(m.NoConstructorNew) assert cstats.alive() == 1 @@ -176,7 +182,6 @@ def test_inheritance(msg): def test_inheritance_init(msg): - # Single base class Python(m.Pet): def __init__(self): @@ -213,7 +218,7 @@ def test_automatic_upcasting(): def test_isinstance(): - objects = [tuple(), dict(), m.Pet("Polly", "parrot")] + [m.Dog("Molly")] * 4 + objects = [(), {}, m.Pet("Polly", "parrot")] + [m.Dog("Molly")] * 4 expected = (True, True, True, True, True, False, False) assert m.check_instances(objects) == expected @@ -313,6 +318,8 @@ def test_bind_protected_functions(): b = m.ProtectedB() assert b.foo() == 42 + assert m.read_foo(b.void_foo()) == 42 + assert m.pointers_equal(b.get_self(), b) class C(m.ProtectedB): def __init__(self): @@ -417,7 +424,7 @@ def test_exception_rvalue_abort(): # https://github.com/pybind/pybind11/issues/1568 -def test_multiple_instances_with_same_pointer(capture): +def test_multiple_instances_with_same_pointer(): n = 100 instances = [m.SamePointer() for _ in range(n)] for i in range(n): @@ -469,3 +476,10 @@ class ClassScope: m.register_duplicate_nested_class_type(ClassScope) expected = 'generic_type: type "YetAnotherDuplicateNested" is already registered!' assert str(exc_info.value) == expected + + +def test_pr4220_tripped_over_this(): + assert ( + m.Empty0().get_msg() + == "This is really only meant to exercise successful compilation." + ) diff --git a/pybind11/tests/test_cmake_build/CMakeLists.txt b/pybind11/tests/test_cmake_build/CMakeLists.txt index 8bfaa38..e5aa975 100644 --- a/pybind11/tests/test_cmake_build/CMakeLists.txt +++ b/pybind11/tests/test_cmake_build/CMakeLists.txt @@ -1,6 +1,3 @@ -# Built-in in CMake 3.5+ -include(CMakeParseArguments) - add_custom_target(test_cmake_build) function(pybind11_add_build_test name) diff --git a/pybind11/tests/test_cmake_build/installed_embed/CMakeLists.txt b/pybind11/tests/test_cmake_build/installed_embed/CMakeLists.txt index f7d6939..d9dcb45 100644 --- a/pybind11/tests/test_cmake_build/installed_embed/CMakeLists.txt +++ b/pybind11/tests/test_cmake_build/installed_embed/CMakeLists.txt @@ -1,12 +1,12 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) -# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.18) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.18) + cmake_policy(VERSION 3.26) endif() project(test_installed_embed CXX) diff --git a/pybind11/tests/test_cmake_build/installed_function/CMakeLists.txt b/pybind11/tests/test_cmake_build/installed_function/CMakeLists.txt index d7ca4db..2f4f642 100644 --- a/pybind11/tests/test_cmake_build/installed_function/CMakeLists.txt +++ b/pybind11/tests/test_cmake_build/installed_function/CMakeLists.txt @@ -1,13 +1,13 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) project(test_installed_module CXX) -# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.18) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.18) + cmake_policy(VERSION 3.26) endif() project(test_installed_function CXX) diff --git a/pybind11/tests/test_cmake_build/installed_target/CMakeLists.txt b/pybind11/tests/test_cmake_build/installed_target/CMakeLists.txt index bc5e101..a981e23 100644 --- a/pybind11/tests/test_cmake_build/installed_target/CMakeLists.txt +++ b/pybind11/tests/test_cmake_build/installed_target/CMakeLists.txt @@ -1,12 +1,12 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) -# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.18) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.18) + cmake_policy(VERSION 3.26) endif() project(test_installed_target CXX) diff --git a/pybind11/tests/test_cmake_build/subdirectory_embed/CMakeLists.txt b/pybind11/tests/test_cmake_build/subdirectory_embed/CMakeLists.txt index 58cdd7c..f286746 100644 --- a/pybind11/tests/test_cmake_build/subdirectory_embed/CMakeLists.txt +++ b/pybind11/tests/test_cmake_build/subdirectory_embed/CMakeLists.txt @@ -1,12 +1,12 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) -# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.18) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.18) + cmake_policy(VERSION 3.26) endif() project(test_subdirectory_embed CXX) diff --git a/pybind11/tests/test_cmake_build/subdirectory_function/CMakeLists.txt b/pybind11/tests/test_cmake_build/subdirectory_function/CMakeLists.txt index 01557c4..275a75c 100644 --- a/pybind11/tests/test_cmake_build/subdirectory_function/CMakeLists.txt +++ b/pybind11/tests/test_cmake_build/subdirectory_function/CMakeLists.txt @@ -1,12 +1,12 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) -# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.18) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.18) + cmake_policy(VERSION 3.26) endif() project(test_subdirectory_function CXX) diff --git a/pybind11/tests/test_cmake_build/subdirectory_target/CMakeLists.txt b/pybind11/tests/test_cmake_build/subdirectory_target/CMakeLists.txt index ba82fde..37bb2c5 100644 --- a/pybind11/tests/test_cmake_build/subdirectory_target/CMakeLists.txt +++ b/pybind11/tests/test_cmake_build/subdirectory_target/CMakeLists.txt @@ -1,12 +1,12 @@ -cmake_minimum_required(VERSION 3.4) +cmake_minimum_required(VERSION 3.5) -# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.26)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.18) +if(${CMAKE_VERSION} VERSION_LESS 3.26) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.18) + cmake_policy(VERSION 3.26) endif() project(test_subdirectory_target CXX) diff --git a/pybind11/tests/test_const_name.py b/pybind11/tests/test_const_name.py index 10b0cae..a145f0b 100644 --- a/pybind11/tests/test_const_name.py +++ b/pybind11/tests/test_const_name.py @@ -3,9 +3,9 @@ from pybind11_tests import const_name as m -@pytest.mark.parametrize("func", (m.const_name_tests, m.underscore_tests)) +@pytest.mark.parametrize("func", [m.const_name_tests, m.underscore_tests]) @pytest.mark.parametrize( - "selector, expected", + ("selector", "expected"), enumerate( ( "", diff --git a/pybind11/tests/test_constants_and_functions.cpp b/pybind11/tests/test_constants_and_functions.cpp index 1918a42..312edca 100644 --- a/pybind11/tests/test_constants_and_functions.cpp +++ b/pybind11/tests/test_constants_and_functions.cpp @@ -52,15 +52,12 @@ int f1(int x) noexcept { return x + 1; } #endif int f2(int x) noexcept(true) { return x + 2; } int f3(int x) noexcept(false) { return x + 3; } -#if defined(__GNUG__) && !defined(__INTEL_COMPILER) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wdeprecated" -#endif +PYBIND11_WARNING_PUSH +PYBIND11_WARNING_DISABLE_GCC("-Wdeprecated") +PYBIND11_WARNING_DISABLE_CLANG("-Wdeprecated") // NOLINTNEXTLINE(modernize-use-noexcept) int f4(int x) throw() { return x + 4; } // Deprecated equivalent to noexcept(true) -#if defined(__GNUG__) && !defined(__INTEL_COMPILER) -# pragma GCC diagnostic pop -#endif +PYBIND11_WARNING_POP struct C { int m1(int x) noexcept { return x - 1; } int m2(int x) const noexcept { return x - 2; } @@ -68,17 +65,14 @@ struct C { int m4(int x) const noexcept(true) { return x - 4; } int m5(int x) noexcept(false) { return x - 5; } int m6(int x) const noexcept(false) { return x - 6; } -#if defined(__GNUG__) && !defined(__INTEL_COMPILER) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wdeprecated" -#endif + PYBIND11_WARNING_PUSH + PYBIND11_WARNING_DISABLE_GCC("-Wdeprecated") + PYBIND11_WARNING_DISABLE_CLANG("-Wdeprecated") // NOLINTNEXTLINE(modernize-use-noexcept) int m7(int x) throw() { return x - 7; } // NOLINTNEXTLINE(modernize-use-noexcept) int m8(int x) const throw() { return x - 8; } -#if defined(__GNUG__) && !defined(__INTEL_COMPILER) -# pragma GCC diagnostic pop -#endif + PYBIND11_WARNING_POP }; } // namespace test_exc_sp @@ -126,14 +120,12 @@ TEST_SUBMODULE(constants_and_functions, m) { .def("m8", &C::m8); m.def("f1", f1); m.def("f2", f2); -#if defined(__INTEL_COMPILER) -# pragma warning push -# pragma warning disable 878 // incompatible exception specifications -#endif + + PYBIND11_WARNING_PUSH + PYBIND11_WARNING_DISABLE_INTEL(878) // incompatible exception specifications m.def("f3", f3); -#if defined(__INTEL_COMPILER) -# pragma warning pop -#endif + PYBIND11_WARNING_POP + m.def("f4", f4); // test_function_record_leaks @@ -156,4 +148,7 @@ TEST_SUBMODULE(constants_and_functions, m) { py::arg_v("y", 42, ""), py::arg_v("z", default_value)); }); + + // test noexcept(true) lambda (#4565) + m.def("l1", []() noexcept(true) { return 0; }); } diff --git a/pybind11/tests/test_constants_and_functions.py b/pybind11/tests/test_constants_and_functions.py index 5da0b84..a114246 100644 --- a/pybind11/tests/test_constants_and_functions.py +++ b/pybind11/tests/test_constants_and_functions.py @@ -50,3 +50,7 @@ def __repr__(self): m.register_large_capture_with_invalid_arguments(m) with pytest.raises(RuntimeError): m.register_with_raising_repr(m, RaisingRepr()) + + +def test_noexcept_lambda(): + assert m.l1() == 0 diff --git a/pybind11/tests/test_copy_move.cpp b/pybind11/tests/test_copy_move.cpp index 28c2445..f547335 100644 --- a/pybind11/tests/test_copy_move.cpp +++ b/pybind11/tests/test_copy_move.cpp @@ -13,6 +13,8 @@ #include "constructor_stats.h" #include "pybind11_tests.h" +#include + template struct empty { static const derived &get_one() { return instance_; } @@ -293,3 +295,239 @@ TEST_SUBMODULE(copy_move_policies, m) { // Make sure that cast from pytype rvalue to other pytype works m.def("get_pytype_rvalue_castissue", [](double i) { return py::float_(i).cast(); }); } + +/* + * Rest of the file: + * static_assert based tests for pybind11 adaptations of + * std::is_move_constructible, std::is_copy_constructible and + * std::is_copy_assignable (no adaptation of std::is_move_assignable). + * Difference between pybind11 and std traits: pybind11 traits will also check + * the contained value_types. + */ + +struct NotMovable { + NotMovable() = default; + NotMovable(NotMovable const &) = default; + NotMovable(NotMovable &&) = delete; + NotMovable &operator=(NotMovable const &) = default; + NotMovable &operator=(NotMovable &&) = delete; +}; +static_assert(!std::is_move_constructible::value, + "!std::is_move_constructible::value"); +static_assert(std::is_copy_constructible::value, + "std::is_copy_constructible::value"); +static_assert(!pybind11::detail::is_move_constructible::value, + "!pybind11::detail::is_move_constructible::value"); +static_assert(pybind11::detail::is_copy_constructible::value, + "pybind11::detail::is_copy_constructible::value"); +static_assert(!std::is_move_assignable::value, + "!std::is_move_assignable::value"); +static_assert(std::is_copy_assignable::value, + "std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(pybind11::detail::is_copy_assignable::value, + "pybind11::detail::is_copy_assignable::value"); + +struct NotCopyable { + NotCopyable() = default; + NotCopyable(NotCopyable const &) = delete; + NotCopyable(NotCopyable &&) = default; + NotCopyable &operator=(NotCopyable const &) = delete; + NotCopyable &operator=(NotCopyable &&) = default; +}; +static_assert(std::is_move_constructible::value, + "std::is_move_constructible::value"); +static_assert(!std::is_copy_constructible::value, + "!std::is_copy_constructible::value"); +static_assert(pybind11::detail::is_move_constructible::value, + "pybind11::detail::is_move_constructible::value"); +static_assert(!pybind11::detail::is_copy_constructible::value, + "!pybind11::detail::is_copy_constructible::value"); +static_assert(std::is_move_assignable::value, + "std::is_move_assignable::value"); +static_assert(!std::is_copy_assignable::value, + "!std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(!pybind11::detail::is_copy_assignable::value, + "!pybind11::detail::is_copy_assignable::value"); + +struct NotCopyableNotMovable { + NotCopyableNotMovable() = default; + NotCopyableNotMovable(NotCopyableNotMovable const &) = delete; + NotCopyableNotMovable(NotCopyableNotMovable &&) = delete; + NotCopyableNotMovable &operator=(NotCopyableNotMovable const &) = delete; + NotCopyableNotMovable &operator=(NotCopyableNotMovable &&) = delete; +}; +static_assert(!std::is_move_constructible::value, + "!std::is_move_constructible::value"); +static_assert(!std::is_copy_constructible::value, + "!std::is_copy_constructible::value"); +static_assert(!pybind11::detail::is_move_constructible::value, + "!pybind11::detail::is_move_constructible::value"); +static_assert(!pybind11::detail::is_copy_constructible::value, + "!pybind11::detail::is_copy_constructible::value"); +static_assert(!std::is_move_assignable::value, + "!std::is_move_assignable::value"); +static_assert(!std::is_copy_assignable::value, + "!std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(!pybind11::detail::is_copy_assignable::value, + "!pybind11::detail::is_copy_assignable::value"); + +struct NotMovableVector : std::vector {}; +static_assert(std::is_move_constructible::value, + "std::is_move_constructible::value"); +static_assert(std::is_copy_constructible::value, + "std::is_copy_constructible::value"); +static_assert(!pybind11::detail::is_move_constructible::value, + "!pybind11::detail::is_move_constructible::value"); +static_assert(pybind11::detail::is_copy_constructible::value, + "pybind11::detail::is_copy_constructible::value"); +static_assert(std::is_move_assignable::value, + "std::is_move_assignable::value"); +static_assert(std::is_copy_assignable::value, + "std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(pybind11::detail::is_copy_assignable::value, + "pybind11::detail::is_copy_assignable::value"); + +struct NotCopyableVector : std::vector {}; +static_assert(std::is_move_constructible::value, + "std::is_move_constructible::value"); +static_assert(std::is_copy_constructible::value, + "std::is_copy_constructible::value"); +static_assert(pybind11::detail::is_move_constructible::value, + "pybind11::detail::is_move_constructible::value"); +static_assert(!pybind11::detail::is_copy_constructible::value, + "!pybind11::detail::is_copy_constructible::value"); +static_assert(std::is_move_assignable::value, + "std::is_move_assignable::value"); +static_assert(std::is_copy_assignable::value, + "std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(!pybind11::detail::is_copy_assignable::value, + "!pybind11::detail::is_copy_assignable::value"); + +struct NotCopyableNotMovableVector : std::vector {}; +static_assert(std::is_move_constructible::value, + "std::is_move_constructible::value"); +static_assert(std::is_copy_constructible::value, + "std::is_copy_constructible::value"); +static_assert(!pybind11::detail::is_move_constructible::value, + "!pybind11::detail::is_move_constructible::value"); +static_assert(!pybind11::detail::is_copy_constructible::value, + "!pybind11::detail::is_copy_constructible::value"); +static_assert(std::is_move_assignable::value, + "std::is_move_assignable::value"); +static_assert(std::is_copy_assignable::value, + "std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(!pybind11::detail::is_copy_assignable::value, + "!pybind11::detail::is_copy_assignable::value"); + +struct NotMovableMap : std::map {}; +static_assert(std::is_move_constructible::value, + "std::is_move_constructible::value"); +static_assert(std::is_copy_constructible::value, + "std::is_copy_constructible::value"); +static_assert(!pybind11::detail::is_move_constructible::value, + "!pybind11::detail::is_move_constructible::value"); +static_assert(pybind11::detail::is_copy_constructible::value, + "pybind11::detail::is_copy_constructible::value"); +static_assert(std::is_move_assignable::value, + "std::is_move_assignable::value"); +static_assert(std::is_copy_assignable::value, + "std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(pybind11::detail::is_copy_assignable::value, + "pybind11::detail::is_copy_assignable::value"); + +struct NotCopyableMap : std::map {}; +static_assert(std::is_move_constructible::value, + "std::is_move_constructible::value"); +static_assert(std::is_copy_constructible::value, + "std::is_copy_constructible::value"); +static_assert(pybind11::detail::is_move_constructible::value, + "pybind11::detail::is_move_constructible::value"); +static_assert(!pybind11::detail::is_copy_constructible::value, + "!pybind11::detail::is_copy_constructible::value"); +static_assert(std::is_move_assignable::value, + "std::is_move_assignable::value"); +static_assert(std::is_copy_assignable::value, + "std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(!pybind11::detail::is_copy_assignable::value, + "!pybind11::detail::is_copy_assignable::value"); + +struct NotCopyableNotMovableMap : std::map {}; +static_assert(std::is_move_constructible::value, + "std::is_move_constructible::value"); +static_assert(std::is_copy_constructible::value, + "std::is_copy_constructible::value"); +static_assert(!pybind11::detail::is_move_constructible::value, + "!pybind11::detail::is_move_constructible::value"); +static_assert(!pybind11::detail::is_copy_constructible::value, + "!pybind11::detail::is_copy_constructible::value"); +static_assert(std::is_move_assignable::value, + "std::is_move_assignable::value"); +static_assert(std::is_copy_assignable::value, + "std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(!pybind11::detail::is_copy_assignable::value, + "!pybind11::detail::is_copy_assignable::value"); + +struct RecursiveVector : std::vector {}; +static_assert(std::is_move_constructible::value, + "std::is_move_constructible::value"); +static_assert(std::is_copy_constructible::value, + "std::is_copy_constructible::value"); +static_assert(pybind11::detail::is_move_constructible::value, + "pybind11::detail::is_move_constructible::value"); +static_assert(pybind11::detail::is_copy_constructible::value, + "pybind11::detail::is_copy_constructible::value"); +static_assert(std::is_move_assignable::value, + "std::is_move_assignable::value"); +static_assert(std::is_copy_assignable::value, + "std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(pybind11::detail::is_copy_assignable::value, + "pybind11::detail::is_copy_assignable::value"); + +struct RecursiveMap : std::map {}; +static_assert(std::is_move_constructible::value, + "std::is_move_constructible::value"); +static_assert(std::is_copy_constructible::value, + "std::is_copy_constructible::value"); +static_assert(pybind11::detail::is_move_constructible::value, + "pybind11::detail::is_move_constructible::value"); +static_assert(pybind11::detail::is_copy_constructible::value, + "pybind11::detail::is_copy_constructible::value"); +static_assert(std::is_move_assignable::value, + "std::is_move_assignable::value"); +static_assert(std::is_copy_assignable::value, + "std::is_copy_assignable::value"); +// pybind11 does not have this +// static_assert(!pybind11::detail::is_move_assignable::value, +// "!pybind11::detail::is_move_assignable::value"); +static_assert(pybind11::detail::is_copy_assignable::value, + "pybind11::detail::is_copy_assignable::value"); diff --git a/pybind11/tests/test_custom_type_casters.cpp b/pybind11/tests/test_custom_type_casters.cpp index 25540e3..b4af02a 100644 --- a/pybind11/tests/test_custom_type_casters.cpp +++ b/pybind11/tests/test_custom_type_casters.cpp @@ -21,7 +21,7 @@ class ArgInspector2 { }; class ArgAlwaysConverts {}; -namespace pybind11 { +namespace PYBIND11_NAMESPACE { namespace detail { template <> struct type_caster { @@ -74,7 +74,7 @@ struct type_caster { } }; } // namespace detail -} // namespace pybind11 +} // namespace PYBIND11_NAMESPACE // test_custom_caster_destruction class DestructionTester { @@ -92,7 +92,7 @@ class DestructionTester { return *this; } }; -namespace pybind11 { +namespace PYBIND11_NAMESPACE { namespace detail { template <> struct type_caster { @@ -104,7 +104,7 @@ struct type_caster { } }; } // namespace detail -} // namespace pybind11 +} // namespace PYBIND11_NAMESPACE // Define type caster outside of `pybind11::detail` and then alias it. namespace other_lib { @@ -112,7 +112,7 @@ struct MyType {}; // Corrupt `py` shorthand alias for surrounding context. namespace py {} // Corrupt unqualified relative `pybind11` namespace. -namespace pybind11 {} +namespace PYBIND11_NAMESPACE {} // Correct alias. namespace py_ = ::pybind11; // Define caster. This is effectively no-op, we only ensure it compiles and we @@ -127,12 +127,12 @@ struct my_caster { }; } // namespace other_lib // Effectively "alias" it into correct namespace (via inheritance). -namespace pybind11 { +namespace PYBIND11_NAMESPACE { namespace detail { template <> struct type_caster : public other_lib::my_caster {}; } // namespace detail -} // namespace pybind11 +} // namespace PYBIND11_NAMESPACE TEST_SUBMODULE(custom_type_casters, m) { // test_custom_type_casters diff --git a/pybind11/tests/test_custom_type_casters.py b/pybind11/tests/test_custom_type_casters.py index adfa6cf..3a00ea9 100644 --- a/pybind11/tests/test_custom_type_casters.py +++ b/pybind11/tests/test_custom_type_casters.py @@ -94,12 +94,14 @@ def test_noconvert_args(msg): def test_custom_caster_destruction(): """Tests that returning a pointer to a type that gets converted with a custom type caster gets - destroyed when the function has py::return_value_policy::take_ownership policy applied.""" + destroyed when the function has py::return_value_policy::take_ownership policy applied. + """ cstats = m.destruction_tester_cstats() # This one *doesn't* have take_ownership: the pointer should be used but not destroyed: z = m.custom_caster_no_destroy() - assert cstats.alive() == 1 and cstats.default_constructions == 1 + assert cstats.alive() == 1 + assert cstats.default_constructions == 1 assert z # take_ownership applied: this constructs a new object, casts it, then destroys it: diff --git a/pybind11/tests/test_custom_type_setup.py b/pybind11/tests/test_custom_type_setup.py index 19b44c9..e63ff57 100644 --- a/pybind11/tests/test_custom_type_setup.py +++ b/pybind11/tests/test_custom_type_setup.py @@ -7,7 +7,7 @@ from pybind11_tests import custom_type_setup as m -@pytest.fixture +@pytest.fixture() def gc_tester(): """Tests that an object is garbage collected. diff --git a/pybind11/tests/test_docstring_options.cpp b/pybind11/tests/test_docstring_options.cpp index 4d44f4e..dda1cf6 100644 --- a/pybind11/tests/test_docstring_options.cpp +++ b/pybind11/tests/test_docstring_options.cpp @@ -85,4 +85,57 @@ TEST_SUBMODULE(docstring_options, m) { &DocstringTestFoo::setValue, "This is a property docstring"); } + + { + enum class DocstringTestEnum1 { Member1, Member2 }; + + py::enum_(m, "DocstringTestEnum1", "Enum docstring") + .value("Member1", DocstringTestEnum1::Member1) + .value("Member2", DocstringTestEnum1::Member2); + } + + { + py::options options; + options.enable_enum_members_docstring(); + + enum class DocstringTestEnum2 { Member1, Member2 }; + + py::enum_(m, "DocstringTestEnum2", "Enum docstring") + .value("Member1", DocstringTestEnum2::Member1) + .value("Member2", DocstringTestEnum2::Member2); + } + + { + py::options options; + options.disable_enum_members_docstring(); + + enum class DocstringTestEnum3 { Member1, Member2 }; + + py::enum_(m, "DocstringTestEnum3", "Enum docstring") + .value("Member1", DocstringTestEnum3::Member1) + .value("Member2", DocstringTestEnum3::Member2); + } + + { + py::options options; + options.disable_user_defined_docstrings(); + + enum class DocstringTestEnum4 { Member1, Member2 }; + + py::enum_(m, "DocstringTestEnum4", "Enum docstring") + .value("Member1", DocstringTestEnum4::Member1) + .value("Member2", DocstringTestEnum4::Member2); + } + + { + py::options options; + options.disable_user_defined_docstrings(); + options.disable_enum_members_docstring(); + + enum class DocstringTestEnum5 { Member1, Member2 }; + + py::enum_(m, "DocstringTestEnum5", "Enum docstring") + .value("Member1", DocstringTestEnum5::Member1) + .value("Member2", DocstringTestEnum5::Member2); + } } diff --git a/pybind11/tests/test_docstring_options.py b/pybind11/tests/test_docstring_options.py index fcd16b8..e6f5a9d 100644 --- a/pybind11/tests/test_docstring_options.py +++ b/pybind11/tests/test_docstring_options.py @@ -39,3 +39,26 @@ def test_docstring_options(): # Suppression of user-defined docstrings for non-function objects assert not m.DocstringTestFoo.__doc__ assert not m.DocstringTestFoo.value_prop.__doc__ + + # Check existig behaviour of enum docstings + assert ( + m.DocstringTestEnum1.__doc__ + == "Enum docstring\n\nMembers:\n\n Member1\n\n Member2" + ) + + # options.enable_enum_members_docstring() + assert ( + m.DocstringTestEnum2.__doc__ + == "Enum docstring\n\nMembers:\n\n Member1\n\n Member2" + ) + + # options.disable_enum_members_docstring() + assert m.DocstringTestEnum3.__doc__ == "Enum docstring" + + # options.disable_user_defined_docstrings() + assert m.DocstringTestEnum4.__doc__ == "Members:\n\n Member1\n\n Member2" + + # options.disable_user_defined_docstrings() + # options.disable_enum_members_docstring() + # When all options are disabled, no docstring (instead of an empty one) should be generated + assert m.DocstringTestEnum5.__doc__ is None diff --git a/pybind11/tests/test_eigen_matrix.cpp b/pybind11/tests/test_eigen_matrix.cpp new file mode 100644 index 0000000..554cc4d --- /dev/null +++ b/pybind11/tests/test_eigen_matrix.cpp @@ -0,0 +1,428 @@ +/* + tests/eigen.cpp -- automatic conversion of Eigen types + + Copyright (c) 2016 Wenzel Jakob + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#include +#include + +#include "constructor_stats.h" +#include "pybind11_tests.h" + +PYBIND11_WARNING_DISABLE_MSVC(4996) + +#include + +using MatrixXdR = Eigen::Matrix; + +// Sets/resets a testing reference matrix to have values of 10*r + c, where r and c are the +// (1-based) row/column number. +template +void reset_ref(M &x) { + for (int i = 0; i < x.rows(); i++) { + for (int j = 0; j < x.cols(); j++) { + x(i, j) = 11 + 10 * i + j; + } + } +} + +// Returns a static, column-major matrix +Eigen::MatrixXd &get_cm() { + static Eigen::MatrixXd *x; + if (!x) { + x = new Eigen::MatrixXd(3, 3); + reset_ref(*x); + } + return *x; +} +// Likewise, but row-major +MatrixXdR &get_rm() { + static MatrixXdR *x; + if (!x) { + x = new MatrixXdR(3, 3); + reset_ref(*x); + } + return *x; +} +// Resets the values of the static matrices returned by get_cm()/get_rm() +void reset_refs() { + reset_ref(get_cm()); + reset_ref(get_rm()); +} + +// Returns element 2,1 from a matrix (used to test copy/nocopy) +double get_elem(const Eigen::Ref &m) { return m(2, 1); }; + +// Returns a matrix with 10*r + 100*c added to each matrix element (to help test that the matrix +// reference is referencing rows/columns correctly). +template +Eigen::MatrixXd adjust_matrix(MatrixArgType m) { + Eigen::MatrixXd ret(m); + for (int c = 0; c < m.cols(); c++) { + for (int r = 0; r < m.rows(); r++) { + ret(r, c) += 10 * r + 100 * c; // NOLINT(clang-analyzer-core.uninitialized.Assign) + } + } + return ret; +} + +struct CustomOperatorNew { + CustomOperatorNew() = default; + + Eigen::Matrix4d a = Eigen::Matrix4d::Zero(); + Eigen::Matrix4d b = Eigen::Matrix4d::Identity(); + + EIGEN_MAKE_ALIGNED_OPERATOR_NEW; +}; + +TEST_SUBMODULE(eigen_matrix, m) { + using FixedMatrixR = Eigen::Matrix; + using FixedMatrixC = Eigen::Matrix; + using DenseMatrixR = Eigen::Matrix; + using DenseMatrixC = Eigen::Matrix; + using FourRowMatrixC = Eigen::Matrix; + using FourColMatrixC = Eigen::Matrix; + using FourRowMatrixR = Eigen::Matrix; + using FourColMatrixR = Eigen::Matrix; + using SparseMatrixR = Eigen::SparseMatrix; + using SparseMatrixC = Eigen::SparseMatrix; + + // various tests + m.def("double_col", [](const Eigen::VectorXf &x) -> Eigen::VectorXf { return 2.0f * x; }); + m.def("double_row", + [](const Eigen::RowVectorXf &x) -> Eigen::RowVectorXf { return 2.0f * x; }); + m.def("double_complex", + [](const Eigen::VectorXcf &x) -> Eigen::VectorXcf { return 2.0f * x; }); + m.def("double_threec", [](py::EigenDRef x) { x *= 2; }); + m.def("double_threer", [](py::EigenDRef x) { x *= 2; }); + m.def("double_mat_cm", [](const Eigen::MatrixXf &x) -> Eigen::MatrixXf { return 2.0f * x; }); + m.def("double_mat_rm", [](const DenseMatrixR &x) -> DenseMatrixR { return 2.0f * x; }); + + // test_eigen_ref_to_python + // Different ways of passing via Eigen::Ref; the first and second are the Eigen-recommended + m.def("cholesky1", + [](const Eigen::Ref &x) -> Eigen::MatrixXd { return x.llt().matrixL(); }); + m.def("cholesky2", [](const Eigen::Ref &x) -> Eigen::MatrixXd { + return x.llt().matrixL(); + }); + m.def("cholesky3", + [](const Eigen::Ref &x) -> Eigen::MatrixXd { return x.llt().matrixL(); }); + m.def("cholesky4", [](const Eigen::Ref &x) -> Eigen::MatrixXd { + return x.llt().matrixL(); + }); + + // test_eigen_ref_mutators + // Mutators: these add some value to the given element using Eigen, but Eigen should be mapping + // into the numpy array data and so the result should show up there. There are three versions: + // one that works on a contiguous-row matrix (numpy's default), one for a contiguous-column + // matrix, and one for any matrix. + auto add_rm = [](Eigen::Ref x, int r, int c, double v) { x(r, c) += v; }; + auto add_cm = [](Eigen::Ref x, int r, int c, double v) { x(r, c) += v; }; + + // Mutators (Eigen maps into numpy variables): + m.def("add_rm", add_rm); // Only takes row-contiguous + m.def("add_cm", add_cm); // Only takes column-contiguous + // Overloaded versions that will accept either row or column contiguous: + m.def("add1", add_rm); + m.def("add1", add_cm); + m.def("add2", add_cm); + m.def("add2", add_rm); + // This one accepts a matrix of any stride: + m.def("add_any", + [](py::EigenDRef x, int r, int c, double v) { x(r, c) += v; }); + + // Return mutable references (numpy maps into eigen variables) + m.def("get_cm_ref", []() { return Eigen::Ref(get_cm()); }); + m.def("get_rm_ref", []() { return Eigen::Ref(get_rm()); }); + // The same references, but non-mutable (numpy maps into eigen variables, but is !writeable) + m.def("get_cm_const_ref", []() { return Eigen::Ref(get_cm()); }); + m.def("get_rm_const_ref", []() { return Eigen::Ref(get_rm()); }); + + m.def("reset_refs", reset_refs); // Restores get_{cm,rm}_ref to original values + + // Increments and returns ref to (same) matrix + m.def( + "incr_matrix", + [](Eigen::Ref m, double v) { + m += Eigen::MatrixXd::Constant(m.rows(), m.cols(), v); + return m; + }, + py::return_value_policy::reference); + + // Same, but accepts a matrix of any strides + m.def( + "incr_matrix_any", + [](py::EigenDRef m, double v) { + m += Eigen::MatrixXd::Constant(m.rows(), m.cols(), v); + return m; + }, + py::return_value_policy::reference); + + // Returns an eigen slice of even rows + m.def( + "even_rows", + [](py::EigenDRef m) { + return py::EigenDMap( + m.data(), + (m.rows() + 1) / 2, + m.cols(), + py::EigenDStride(m.outerStride(), 2 * m.innerStride())); + }, + py::return_value_policy::reference); + + // Returns an eigen slice of even columns + m.def( + "even_cols", + [](py::EigenDRef m) { + return py::EigenDMap( + m.data(), + m.rows(), + (m.cols() + 1) / 2, + py::EigenDStride(2 * m.outerStride(), m.innerStride())); + }, + py::return_value_policy::reference); + + // Returns diagonals: a vector-like object with an inner stride != 1 + m.def("diagonal", [](const Eigen::Ref &x) { return x.diagonal(); }); + m.def("diagonal_1", + [](const Eigen::Ref &x) { return x.diagonal<1>(); }); + m.def("diagonal_n", + [](const Eigen::Ref &x, int index) { return x.diagonal(index); }); + + // Return a block of a matrix (gives non-standard strides) + m.def("block", + [m](const py::object &x_obj, + int start_row, + int start_col, + int block_rows, + int block_cols) { + return m.attr("_block")(x_obj, x_obj, start_row, start_col, block_rows, block_cols); + }); + + m.def( + "_block", + [](const py::object &x_obj, + const Eigen::Ref &x, + int start_row, + int start_col, + int block_rows, + int block_cols) { + // See PR #4217 for background. This test is a bit over the top, but might be useful + // as a concrete example to point to when explaining the dangling reference trap. + auto i0 = py::make_tuple(0, 0); + auto x0_orig = x_obj[*i0].cast(); + if (x(0, 0) != x0_orig) { + throw std::runtime_error( + "Something in the type_caster for Eigen::Ref is terribly wrong."); + } + double x0_mod = x0_orig + 1; + x_obj[*i0] = x0_mod; + auto copy_detected = (x(0, 0) != x0_mod); + x_obj[*i0] = x0_orig; + if (copy_detected) { + throw std::runtime_error("type_caster for Eigen::Ref made a copy."); + } + return x.block(start_row, start_col, block_rows, block_cols); + }, + py::keep_alive<0, 1>()); + + // test_eigen_return_references, test_eigen_keepalive + // return value referencing/copying tests: + class ReturnTester { + Eigen::MatrixXd mat = create(); + + public: + ReturnTester() { print_created(this); } + ~ReturnTester() { print_destroyed(this); } + static Eigen::MatrixXd create() { return Eigen::MatrixXd::Ones(10, 10); } + // NOLINTNEXTLINE(readability-const-return-type) + static const Eigen::MatrixXd createConst() { return Eigen::MatrixXd::Ones(10, 10); } + Eigen::MatrixXd &get() { return mat; } + Eigen::MatrixXd *getPtr() { return &mat; } + const Eigen::MatrixXd &view() { return mat; } + const Eigen::MatrixXd *viewPtr() { return &mat; } + Eigen::Ref ref() { return mat; } + Eigen::Ref refConst() { return mat; } + Eigen::Block block(int r, int c, int nrow, int ncol) { + return mat.block(r, c, nrow, ncol); + } + Eigen::Block blockConst(int r, int c, int nrow, int ncol) const { + return mat.block(r, c, nrow, ncol); + } + py::EigenDMap corners() { + return py::EigenDMap( + mat.data(), + py::EigenDStride(mat.outerStride() * (mat.outerSize() - 1), + mat.innerStride() * (mat.innerSize() - 1))); + } + py::EigenDMap cornersConst() const { + return py::EigenDMap( + mat.data(), + py::EigenDStride(mat.outerStride() * (mat.outerSize() - 1), + mat.innerStride() * (mat.innerSize() - 1))); + } + }; + using rvp = py::return_value_policy; + py::class_(m, "ReturnTester") + .def(py::init<>()) + .def_static("create", &ReturnTester::create) + .def_static("create_const", &ReturnTester::createConst) + .def("get", &ReturnTester::get, rvp::reference_internal) + .def("get_ptr", &ReturnTester::getPtr, rvp::reference_internal) + .def("view", &ReturnTester::view, rvp::reference_internal) + .def("view_ptr", &ReturnTester::view, rvp::reference_internal) + .def("copy_get", &ReturnTester::get) // Default rvp: copy + .def("copy_view", &ReturnTester::view) // " + .def("ref", &ReturnTester::ref) // Default for Ref is to reference + .def("ref_const", &ReturnTester::refConst) // Likewise, but const + .def("ref_safe", &ReturnTester::ref, rvp::reference_internal) + .def("ref_const_safe", &ReturnTester::refConst, rvp::reference_internal) + .def("copy_ref", &ReturnTester::ref, rvp::copy) + .def("copy_ref_const", &ReturnTester::refConst, rvp::copy) + .def("block", &ReturnTester::block) + .def("block_safe", &ReturnTester::block, rvp::reference_internal) + .def("block_const", &ReturnTester::blockConst, rvp::reference_internal) + .def("copy_block", &ReturnTester::block, rvp::copy) + .def("corners", &ReturnTester::corners, rvp::reference_internal) + .def("corners_const", &ReturnTester::cornersConst, rvp::reference_internal); + + // test_special_matrix_objects + // Returns a DiagonalMatrix with diagonal (1,2,3,...) + m.def("incr_diag", [](int k) { + Eigen::DiagonalMatrix m(k); + for (int i = 0; i < k; i++) { + m.diagonal()[i] = i + 1; + } + return m; + }); + + // Returns a SelfAdjointView referencing the lower triangle of m + m.def("symmetric_lower", + [](const Eigen::MatrixXi &m) { return m.selfadjointView(); }); + // Returns a SelfAdjointView referencing the lower triangle of m + m.def("symmetric_upper", + [](const Eigen::MatrixXi &m) { return m.selfadjointView(); }); + + // Test matrix for various functions below. + Eigen::MatrixXf mat(5, 6); + mat << 0, 3, 0, 0, 0, 11, 22, 0, 0, 0, 17, 11, 7, 5, 0, 1, 0, 11, 0, 0, 0, 0, 0, 11, 0, 0, 14, + 0, 8, 11; + + // test_fixed, and various other tests + m.def("fixed_r", [mat]() -> FixedMatrixR { return FixedMatrixR(mat); }); + // Our Eigen does a hack which respects constness through the numpy writeable flag. + // Therefore, the const return actually affects this type despite being an rvalue. + // NOLINTNEXTLINE(readability-const-return-type) + m.def("fixed_r_const", [mat]() -> const FixedMatrixR { return FixedMatrixR(mat); }); + m.def("fixed_c", [mat]() -> FixedMatrixC { return FixedMatrixC(mat); }); + m.def("fixed_copy_r", [](const FixedMatrixR &m) -> FixedMatrixR { return m; }); + m.def("fixed_copy_c", [](const FixedMatrixC &m) -> FixedMatrixC { return m; }); + // test_mutator_descriptors + m.def("fixed_mutator_r", [](const Eigen::Ref &) {}); + m.def("fixed_mutator_c", [](const Eigen::Ref &) {}); + m.def("fixed_mutator_a", [](const py::EigenDRef &) {}); + // test_dense + m.def("dense_r", [mat]() -> DenseMatrixR { return DenseMatrixR(mat); }); + m.def("dense_c", [mat]() -> DenseMatrixC { return DenseMatrixC(mat); }); + m.def("dense_copy_r", [](const DenseMatrixR &m) -> DenseMatrixR { return m; }); + m.def("dense_copy_c", [](const DenseMatrixC &m) -> DenseMatrixC { return m; }); + // test_sparse, test_sparse_signature + m.def("sparse_r", [mat]() -> SparseMatrixR { + // NOLINTNEXTLINE(clang-analyzer-core.uninitialized.UndefReturn) + return Eigen::SparseView(mat); + }); + m.def("sparse_c", + [mat]() -> SparseMatrixC { return Eigen::SparseView(mat); }); + m.def("sparse_copy_r", [](const SparseMatrixR &m) -> SparseMatrixR { return m; }); + m.def("sparse_copy_c", [](const SparseMatrixC &m) -> SparseMatrixC { return m; }); + // test_partially_fixed + m.def("partial_copy_four_rm_r", [](const FourRowMatrixR &m) -> FourRowMatrixR { return m; }); + m.def("partial_copy_four_rm_c", [](const FourColMatrixR &m) -> FourColMatrixR { return m; }); + m.def("partial_copy_four_cm_r", [](const FourRowMatrixC &m) -> FourRowMatrixC { return m; }); + m.def("partial_copy_four_cm_c", [](const FourColMatrixC &m) -> FourColMatrixC { return m; }); + + // test_cpp_casting + // Test that we can cast a numpy object to a Eigen::MatrixXd explicitly + m.def("cpp_copy", [](py::handle m) { return m.cast()(1, 0); }); + m.def("cpp_ref_c", [](py::handle m) { return m.cast>()(1, 0); }); + m.def("cpp_ref_r", [](py::handle m) { return m.cast>()(1, 0); }); + m.def("cpp_ref_any", + [](py::handle m) { return m.cast>()(1, 0); }); + + // [workaround(intel)] ICC 20/21 breaks with py::arg().stuff, using py::arg{}.stuff works. + + // test_nocopy_wrapper + // Test that we can prevent copying into an argument that would normally copy: First a version + // that would allow copying (if types or strides don't match) for comparison: + m.def("get_elem", &get_elem); + // Now this alternative that calls the tells pybind to fail rather than copy: + m.def( + "get_elem_nocopy", + [](const Eigen::Ref &m) -> double { return get_elem(m); }, + py::arg{}.noconvert()); + // Also test a row-major-only no-copy const ref: + m.def( + "get_elem_rm_nocopy", + [](Eigen::Ref> &m) -> long { + return m(2, 1); + }, + py::arg{}.noconvert()); + + // test_issue738, test_zero_length + // Issue #738: 1Γ—N or NΓ—1 2D matrices were neither accepted nor properly copied with an + // incompatible stride value on the length-1 dimension--but that should be allowed (without + // requiring a copy!) because the stride value can be safely ignored on a size-1 dimension. + // Similarly, 0Γ—N or NΓ—0 matrices were not accepted--again, these should be allowed since + // they contain no data. This particularly affects numpy β‰₯ 1.23, which sets the strides to + // 0 if any dimension size is 0. + m.def("iss738_f1", + &adjust_matrix &>, + py::arg{}.noconvert()); + m.def("iss738_f2", + &adjust_matrix> &>, + py::arg{}.noconvert()); + + // test_issue1105 + // Issue #1105: when converting from a numpy two-dimensional (Nx1) or (1xN) value into a dense + // eigen Vector or RowVector, the argument would fail to load because the numpy copy would + // fail: numpy won't broadcast a Nx1 into a 1-dimensional vector. + m.def("iss1105_col", [](const Eigen::VectorXd &) { return true; }); + m.def("iss1105_row", [](const Eigen::RowVectorXd &) { return true; }); + + // test_named_arguments + // Make sure named arguments are working properly: + m.def( + "matrix_multiply", + [](const py::EigenDRef &A, + const py::EigenDRef &B) -> Eigen::MatrixXd { + if (A.cols() != B.rows()) { + throw std::domain_error("Nonconformable matrices!"); + } + return A * B; + }, + py::arg("A"), + py::arg("B")); + + // test_custom_operator_new + py::class_(m, "CustomOperatorNew") + .def(py::init<>()) + .def_readonly("a", &CustomOperatorNew::a) + .def_readonly("b", &CustomOperatorNew::b); + + // test_eigen_ref_life_support + // In case of a failure (the caster's temp array does not live long enough), creating + // a new array (np.ones(10)) increases the chances that the temp array will be garbage + // collected and/or that its memory will be overridden with different values. + m.def("get_elem_direct", [](const Eigen::Ref &v) { + py::module_::import("numpy").attr("ones")(10); + return v(5); + }); + m.def("get_elem_indirect", [](std::vector> v) { + py::module_::import("numpy").attr("ones")(10); + return v[0](5); + }); +} diff --git a/pybind11/tests/test_eigen_matrix.py b/pybind11/tests/test_eigen_matrix.py new file mode 100644 index 0000000..b2e7674 --- /dev/null +++ b/pybind11/tests/test_eigen_matrix.py @@ -0,0 +1,807 @@ +import pytest + +from pybind11_tests import ConstructorStats + +np = pytest.importorskip("numpy") +m = pytest.importorskip("pybind11_tests.eigen_matrix") + + +ref = np.array( + [ + [0.0, 3, 0, 0, 0, 11], + [22, 0, 0, 0, 17, 11], + [7, 5, 0, 1, 0, 11], + [0, 0, 0, 0, 0, 11], + [0, 0, 14, 0, 8, 11], + ] +) + + +def assert_equal_ref(mat): + np.testing.assert_array_equal(mat, ref) + + +def assert_sparse_equal_ref(sparse_mat): + assert_equal_ref(sparse_mat.toarray()) + + +def test_fixed(): + assert_equal_ref(m.fixed_c()) + assert_equal_ref(m.fixed_r()) + assert_equal_ref(m.fixed_copy_r(m.fixed_r())) + assert_equal_ref(m.fixed_copy_c(m.fixed_c())) + assert_equal_ref(m.fixed_copy_r(m.fixed_c())) + assert_equal_ref(m.fixed_copy_c(m.fixed_r())) + + +def test_dense(): + assert_equal_ref(m.dense_r()) + assert_equal_ref(m.dense_c()) + assert_equal_ref(m.dense_copy_r(m.dense_r())) + assert_equal_ref(m.dense_copy_c(m.dense_c())) + assert_equal_ref(m.dense_copy_r(m.dense_c())) + assert_equal_ref(m.dense_copy_c(m.dense_r())) + + +def test_partially_fixed(): + ref2 = np.array([[0.0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11], [12, 13, 14, 15]]) + np.testing.assert_array_equal(m.partial_copy_four_rm_r(ref2), ref2) + np.testing.assert_array_equal(m.partial_copy_four_rm_c(ref2), ref2) + np.testing.assert_array_equal(m.partial_copy_four_rm_r(ref2[:, 1]), ref2[:, [1]]) + np.testing.assert_array_equal(m.partial_copy_four_rm_c(ref2[0, :]), ref2[[0], :]) + np.testing.assert_array_equal( + m.partial_copy_four_rm_r(ref2[:, (0, 2)]), ref2[:, (0, 2)] + ) + np.testing.assert_array_equal( + m.partial_copy_four_rm_c(ref2[(3, 1, 2), :]), ref2[(3, 1, 2), :] + ) + + np.testing.assert_array_equal(m.partial_copy_four_cm_r(ref2), ref2) + np.testing.assert_array_equal(m.partial_copy_four_cm_c(ref2), ref2) + np.testing.assert_array_equal(m.partial_copy_four_cm_r(ref2[:, 1]), ref2[:, [1]]) + np.testing.assert_array_equal(m.partial_copy_four_cm_c(ref2[0, :]), ref2[[0], :]) + np.testing.assert_array_equal( + m.partial_copy_four_cm_r(ref2[:, (0, 2)]), ref2[:, (0, 2)] + ) + np.testing.assert_array_equal( + m.partial_copy_four_cm_c(ref2[(3, 1, 2), :]), ref2[(3, 1, 2), :] + ) + + # TypeError should be raise for a shape mismatch + functions = [ + m.partial_copy_four_rm_r, + m.partial_copy_four_rm_c, + m.partial_copy_four_cm_r, + m.partial_copy_four_cm_c, + ] + matrix_with_wrong_shape = [[1, 2], [3, 4]] + for f in functions: + with pytest.raises(TypeError) as excinfo: + f(matrix_with_wrong_shape) + assert "incompatible function arguments" in str(excinfo.value) + + +def test_mutator_descriptors(): + zr = np.arange(30, dtype="float32").reshape(5, 6) # row-major + zc = zr.reshape(6, 5).transpose() # column-major + + m.fixed_mutator_r(zr) + m.fixed_mutator_c(zc) + m.fixed_mutator_a(zr) + m.fixed_mutator_a(zc) + with pytest.raises(TypeError) as excinfo: + m.fixed_mutator_r(zc) + assert ( + "(arg0: numpy.ndarray[numpy.float32[5, 6]," + " flags.writeable, flags.c_contiguous]) -> None" in str(excinfo.value) + ) + with pytest.raises(TypeError) as excinfo: + m.fixed_mutator_c(zr) + assert ( + "(arg0: numpy.ndarray[numpy.float32[5, 6]," + " flags.writeable, flags.f_contiguous]) -> None" in str(excinfo.value) + ) + with pytest.raises(TypeError) as excinfo: + m.fixed_mutator_a(np.array([[1, 2], [3, 4]], dtype="float32")) + assert "(arg0: numpy.ndarray[numpy.float32[5, 6], flags.writeable]) -> None" in str( + excinfo.value + ) + zr.flags.writeable = False + with pytest.raises(TypeError): + m.fixed_mutator_r(zr) + with pytest.raises(TypeError): + m.fixed_mutator_a(zr) + + +def test_cpp_casting(): + assert m.cpp_copy(m.fixed_r()) == 22.0 + assert m.cpp_copy(m.fixed_c()) == 22.0 + z = np.array([[5.0, 6], [7, 8]]) + assert m.cpp_copy(z) == 7.0 + assert m.cpp_copy(m.get_cm_ref()) == 21.0 + assert m.cpp_copy(m.get_rm_ref()) == 21.0 + assert m.cpp_ref_c(m.get_cm_ref()) == 21.0 + assert m.cpp_ref_r(m.get_rm_ref()) == 21.0 + with pytest.raises(RuntimeError) as excinfo: + # Can't reference m.fixed_c: it contains floats, m.cpp_ref_any wants doubles + m.cpp_ref_any(m.fixed_c()) + assert "Unable to cast Python instance" in str(excinfo.value) + with pytest.raises(RuntimeError) as excinfo: + # Can't reference m.fixed_r: it contains floats, m.cpp_ref_any wants doubles + m.cpp_ref_any(m.fixed_r()) + assert "Unable to cast Python instance" in str(excinfo.value) + assert m.cpp_ref_any(m.ReturnTester.create()) == 1.0 + + assert m.cpp_ref_any(m.get_cm_ref()) == 21.0 + assert m.cpp_ref_any(m.get_cm_ref()) == 21.0 + + +def test_pass_readonly_array(): + z = np.full((5, 6), 42.0) + z.flags.writeable = False + np.testing.assert_array_equal(z, m.fixed_copy_r(z)) + np.testing.assert_array_equal(m.fixed_r_const(), m.fixed_r()) + assert not m.fixed_r_const().flags.writeable + np.testing.assert_array_equal(m.fixed_copy_r(m.fixed_r_const()), m.fixed_r_const()) + + +def test_nonunit_stride_from_python(): + counting_mat = np.arange(9.0, dtype=np.float32).reshape((3, 3)) + second_row = counting_mat[1, :] + second_col = counting_mat[:, 1] + np.testing.assert_array_equal(m.double_row(second_row), 2.0 * second_row) + np.testing.assert_array_equal(m.double_col(second_row), 2.0 * second_row) + np.testing.assert_array_equal(m.double_complex(second_row), 2.0 * second_row) + np.testing.assert_array_equal(m.double_row(second_col), 2.0 * second_col) + np.testing.assert_array_equal(m.double_col(second_col), 2.0 * second_col) + np.testing.assert_array_equal(m.double_complex(second_col), 2.0 * second_col) + + counting_3d = np.arange(27.0, dtype=np.float32).reshape((3, 3, 3)) + slices = [counting_3d[0, :, :], counting_3d[:, 0, :], counting_3d[:, :, 0]] + for ref_mat in slices: + np.testing.assert_array_equal(m.double_mat_cm(ref_mat), 2.0 * ref_mat) + np.testing.assert_array_equal(m.double_mat_rm(ref_mat), 2.0 * ref_mat) + + # Mutator: + m.double_threer(second_row) + m.double_threec(second_col) + np.testing.assert_array_equal(counting_mat, [[0.0, 2, 2], [6, 16, 10], [6, 14, 8]]) + + +def test_negative_stride_from_python(msg): + """Eigen doesn't support (as of yet) negative strides. When a function takes an Eigen matrix by + copy or const reference, we can pass a numpy array that has negative strides. Otherwise, an + exception will be thrown as Eigen will not be able to map the numpy array.""" + + counting_mat = np.arange(9.0, dtype=np.float32).reshape((3, 3)) + counting_mat = counting_mat[::-1, ::-1] + second_row = counting_mat[1, :] + second_col = counting_mat[:, 1] + np.testing.assert_array_equal(m.double_row(second_row), 2.0 * second_row) + np.testing.assert_array_equal(m.double_col(second_row), 2.0 * second_row) + np.testing.assert_array_equal(m.double_complex(second_row), 2.0 * second_row) + np.testing.assert_array_equal(m.double_row(second_col), 2.0 * second_col) + np.testing.assert_array_equal(m.double_col(second_col), 2.0 * second_col) + np.testing.assert_array_equal(m.double_complex(second_col), 2.0 * second_col) + + counting_3d = np.arange(27.0, dtype=np.float32).reshape((3, 3, 3)) + counting_3d = counting_3d[::-1, ::-1, ::-1] + slices = [counting_3d[0, :, :], counting_3d[:, 0, :], counting_3d[:, :, 0]] + for ref_mat in slices: + np.testing.assert_array_equal(m.double_mat_cm(ref_mat), 2.0 * ref_mat) + np.testing.assert_array_equal(m.double_mat_rm(ref_mat), 2.0 * ref_mat) + + # Mutator: + with pytest.raises(TypeError) as excinfo: + m.double_threer(second_row) + assert ( + msg(excinfo.value) + == """ + double_threer(): incompatible function arguments. The following argument types are supported: + 1. (arg0: numpy.ndarray[numpy.float32[1, 3], flags.writeable]) -> None + + Invoked with: """ + + repr(np.array([5.0, 4.0, 3.0], dtype="float32")) + ) + + with pytest.raises(TypeError) as excinfo: + m.double_threec(second_col) + assert ( + msg(excinfo.value) + == """ + double_threec(): incompatible function arguments. The following argument types are supported: + 1. (arg0: numpy.ndarray[numpy.float32[3, 1], flags.writeable]) -> None + + Invoked with: """ + + repr(np.array([7.0, 4.0, 1.0], dtype="float32")) + ) + + +def test_block_runtime_error_type_caster_eigen_ref_made_a_copy(): + with pytest.raises(RuntimeError) as excinfo: + m.block(ref, 0, 0, 0, 0) + assert str(excinfo.value) == "type_caster for Eigen::Ref made a copy." + + +def test_nonunit_stride_to_python(): + assert np.all(m.diagonal(ref) == ref.diagonal()) + assert np.all(m.diagonal_1(ref) == ref.diagonal(1)) + for i in range(-5, 7): + assert np.all(m.diagonal_n(ref, i) == ref.diagonal(i)), f"m.diagonal_n({i})" + + # Must be order="F", otherwise the type_caster will make a copy and + # m.block() will return a dangling reference (heap-use-after-free). + rof = np.asarray(ref, order="F") + assert np.all(m.block(rof, 2, 1, 3, 3) == rof[2:5, 1:4]) + assert np.all(m.block(rof, 1, 4, 4, 2) == rof[1:, 4:]) + assert np.all(m.block(rof, 1, 4, 3, 2) == rof[1:4, 4:]) + + +def test_eigen_ref_to_python(): + chols = [m.cholesky1, m.cholesky2, m.cholesky3, m.cholesky4] + for i, chol in enumerate(chols, start=1): + mymat = chol(np.array([[1.0, 2, 4], [2, 13, 23], [4, 23, 77]])) + assert np.all( + mymat == np.array([[1, 0, 0], [2, 3, 0], [4, 5, 6]]) + ), f"cholesky{i}" + + +def assign_both(a1, a2, r, c, v): + a1[r, c] = v + a2[r, c] = v + + +def array_copy_but_one(a, r, c, v): + z = np.array(a, copy=True) + z[r, c] = v + return z + + +def test_eigen_return_references(): + """Tests various ways of returning references and non-referencing copies""" + + primary = np.ones((10, 10)) + a = m.ReturnTester() + a_get1 = a.get() + assert not a_get1.flags.owndata + assert a_get1.flags.writeable + assign_both(a_get1, primary, 3, 3, 5) + a_get2 = a.get_ptr() + assert not a_get2.flags.owndata + assert a_get2.flags.writeable + assign_both(a_get1, primary, 2, 3, 6) + + a_view1 = a.view() + assert not a_view1.flags.owndata + assert not a_view1.flags.writeable + with pytest.raises(ValueError): + a_view1[2, 3] = 4 + a_view2 = a.view_ptr() + assert not a_view2.flags.owndata + assert not a_view2.flags.writeable + with pytest.raises(ValueError): + a_view2[2, 3] = 4 + + a_copy1 = a.copy_get() + assert a_copy1.flags.owndata + assert a_copy1.flags.writeable + np.testing.assert_array_equal(a_copy1, primary) + a_copy1[7, 7] = -44 # Shouldn't affect anything else + c1want = array_copy_but_one(primary, 7, 7, -44) + a_copy2 = a.copy_view() + assert a_copy2.flags.owndata + assert a_copy2.flags.writeable + np.testing.assert_array_equal(a_copy2, primary) + a_copy2[4, 4] = -22 # Shouldn't affect anything else + c2want = array_copy_but_one(primary, 4, 4, -22) + + a_ref1 = a.ref() + assert not a_ref1.flags.owndata + assert a_ref1.flags.writeable + assign_both(a_ref1, primary, 1, 1, 15) + a_ref2 = a.ref_const() + assert not a_ref2.flags.owndata + assert not a_ref2.flags.writeable + with pytest.raises(ValueError): + a_ref2[5, 5] = 33 + a_ref3 = a.ref_safe() + assert not a_ref3.flags.owndata + assert a_ref3.flags.writeable + assign_both(a_ref3, primary, 0, 7, 99) + a_ref4 = a.ref_const_safe() + assert not a_ref4.flags.owndata + assert not a_ref4.flags.writeable + with pytest.raises(ValueError): + a_ref4[7, 0] = 987654321 + + a_copy3 = a.copy_ref() + assert a_copy3.flags.owndata + assert a_copy3.flags.writeable + np.testing.assert_array_equal(a_copy3, primary) + a_copy3[8, 1] = 11 + c3want = array_copy_but_one(primary, 8, 1, 11) + a_copy4 = a.copy_ref_const() + assert a_copy4.flags.owndata + assert a_copy4.flags.writeable + np.testing.assert_array_equal(a_copy4, primary) + a_copy4[8, 4] = 88 + c4want = array_copy_but_one(primary, 8, 4, 88) + + a_block1 = a.block(3, 3, 2, 2) + assert not a_block1.flags.owndata + assert a_block1.flags.writeable + a_block1[0, 0] = 55 + primary[3, 3] = 55 + a_block2 = a.block_safe(2, 2, 3, 2) + assert not a_block2.flags.owndata + assert a_block2.flags.writeable + a_block2[2, 1] = -123 + primary[4, 3] = -123 + a_block3 = a.block_const(6, 7, 4, 3) + assert not a_block3.flags.owndata + assert not a_block3.flags.writeable + with pytest.raises(ValueError): + a_block3[2, 2] = -44444 + + a_copy5 = a.copy_block(2, 2, 2, 3) + assert a_copy5.flags.owndata + assert a_copy5.flags.writeable + np.testing.assert_array_equal(a_copy5, primary[2:4, 2:5]) + a_copy5[1, 1] = 777 + c5want = array_copy_but_one(primary[2:4, 2:5], 1, 1, 777) + + a_corn1 = a.corners() + assert not a_corn1.flags.owndata + assert a_corn1.flags.writeable + a_corn1 *= 50 + a_corn1[1, 1] = 999 + primary[0, 0] = 50 + primary[0, 9] = 50 + primary[9, 0] = 50 + primary[9, 9] = 999 + a_corn2 = a.corners_const() + assert not a_corn2.flags.owndata + assert not a_corn2.flags.writeable + with pytest.raises(ValueError): + a_corn2[1, 0] = 51 + + # All of the changes made all the way along should be visible everywhere + # now (except for the copies, of course) + np.testing.assert_array_equal(a_get1, primary) + np.testing.assert_array_equal(a_get2, primary) + np.testing.assert_array_equal(a_view1, primary) + np.testing.assert_array_equal(a_view2, primary) + np.testing.assert_array_equal(a_ref1, primary) + np.testing.assert_array_equal(a_ref2, primary) + np.testing.assert_array_equal(a_ref3, primary) + np.testing.assert_array_equal(a_ref4, primary) + np.testing.assert_array_equal(a_block1, primary[3:5, 3:5]) + np.testing.assert_array_equal(a_block2, primary[2:5, 2:4]) + np.testing.assert_array_equal(a_block3, primary[6:10, 7:10]) + np.testing.assert_array_equal( + a_corn1, primary[0 :: primary.shape[0] - 1, 0 :: primary.shape[1] - 1] + ) + np.testing.assert_array_equal( + a_corn2, primary[0 :: primary.shape[0] - 1, 0 :: primary.shape[1] - 1] + ) + + np.testing.assert_array_equal(a_copy1, c1want) + np.testing.assert_array_equal(a_copy2, c2want) + np.testing.assert_array_equal(a_copy3, c3want) + np.testing.assert_array_equal(a_copy4, c4want) + np.testing.assert_array_equal(a_copy5, c5want) + + +def assert_keeps_alive(cl, method, *args): + cstats = ConstructorStats.get(cl) + start_with = cstats.alive() + a = cl() + assert cstats.alive() == start_with + 1 + z = method(a, *args) + assert cstats.alive() == start_with + 1 + del a + # Here's the keep alive in action: + assert cstats.alive() == start_with + 1 + del z + # Keep alive should have expired: + assert cstats.alive() == start_with + + +def test_eigen_keepalive(): + a = m.ReturnTester() + cstats = ConstructorStats.get(m.ReturnTester) + assert cstats.alive() == 1 + unsafe = [a.ref(), a.ref_const(), a.block(1, 2, 3, 4)] + copies = [ + a.copy_get(), + a.copy_view(), + a.copy_ref(), + a.copy_ref_const(), + a.copy_block(4, 3, 2, 1), + ] + del a + assert cstats.alive() == 0 + del unsafe + del copies + + for meth in [ + m.ReturnTester.get, + m.ReturnTester.get_ptr, + m.ReturnTester.view, + m.ReturnTester.view_ptr, + m.ReturnTester.ref_safe, + m.ReturnTester.ref_const_safe, + m.ReturnTester.corners, + m.ReturnTester.corners_const, + ]: + assert_keeps_alive(m.ReturnTester, meth) + + for meth in [m.ReturnTester.block_safe, m.ReturnTester.block_const]: + assert_keeps_alive(m.ReturnTester, meth, 4, 3, 2, 1) + + +def test_eigen_ref_mutators(): + """Tests Eigen's ability to mutate numpy values""" + + orig = np.array([[1.0, 2, 3], [4, 5, 6], [7, 8, 9]]) + zr = np.array(orig) + zc = np.array(orig, order="F") + m.add_rm(zr, 1, 0, 100) + assert np.all(zr == np.array([[1.0, 2, 3], [104, 5, 6], [7, 8, 9]])) + m.add_cm(zc, 1, 0, 200) + assert np.all(zc == np.array([[1.0, 2, 3], [204, 5, 6], [7, 8, 9]])) + + m.add_any(zr, 1, 0, 20) + assert np.all(zr == np.array([[1.0, 2, 3], [124, 5, 6], [7, 8, 9]])) + m.add_any(zc, 1, 0, 10) + assert np.all(zc == np.array([[1.0, 2, 3], [214, 5, 6], [7, 8, 9]])) + + # Can't reference a col-major array with a row-major Ref, and vice versa: + with pytest.raises(TypeError): + m.add_rm(zc, 1, 0, 1) + with pytest.raises(TypeError): + m.add_cm(zr, 1, 0, 1) + + # Overloads: + m.add1(zr, 1, 0, -100) + m.add2(zr, 1, 0, -20) + assert np.all(zr == orig) + m.add1(zc, 1, 0, -200) + m.add2(zc, 1, 0, -10) + assert np.all(zc == orig) + + # a non-contiguous slice (this won't work on either the row- or + # column-contiguous refs, but should work for the any) + cornersr = zr[0::2, 0::2] + cornersc = zc[0::2, 0::2] + + assert np.all(cornersr == np.array([[1.0, 3], [7, 9]])) + assert np.all(cornersc == np.array([[1.0, 3], [7, 9]])) + + with pytest.raises(TypeError): + m.add_rm(cornersr, 0, 1, 25) + with pytest.raises(TypeError): + m.add_cm(cornersr, 0, 1, 25) + with pytest.raises(TypeError): + m.add_rm(cornersc, 0, 1, 25) + with pytest.raises(TypeError): + m.add_cm(cornersc, 0, 1, 25) + m.add_any(cornersr, 0, 1, 25) + m.add_any(cornersc, 0, 1, 44) + assert np.all(zr == np.array([[1.0, 2, 28], [4, 5, 6], [7, 8, 9]])) + assert np.all(zc == np.array([[1.0, 2, 47], [4, 5, 6], [7, 8, 9]])) + + # You shouldn't be allowed to pass a non-writeable array to a mutating Eigen method: + zro = zr[0:4, 0:4] + zro.flags.writeable = False + with pytest.raises(TypeError): + m.add_rm(zro, 0, 0, 0) + with pytest.raises(TypeError): + m.add_any(zro, 0, 0, 0) + with pytest.raises(TypeError): + m.add1(zro, 0, 0, 0) + with pytest.raises(TypeError): + m.add2(zro, 0, 0, 0) + + # integer array shouldn't be passable to a double-matrix-accepting mutating func: + zi = np.array([[1, 2], [3, 4]]) + with pytest.raises(TypeError): + m.add_rm(zi) + + +def test_numpy_ref_mutators(): + """Tests numpy mutating Eigen matrices (for returned Eigen::Ref<...>s)""" + + m.reset_refs() # In case another test already changed it + + zc = m.get_cm_ref() + zcro = m.get_cm_const_ref() + zr = m.get_rm_ref() + zrro = m.get_rm_const_ref() + + assert [zc[1, 2], zcro[1, 2], zr[1, 2], zrro[1, 2]] == [23] * 4 + + assert not zc.flags.owndata + assert zc.flags.writeable + assert not zr.flags.owndata + assert zr.flags.writeable + assert not zcro.flags.owndata + assert not zcro.flags.writeable + assert not zrro.flags.owndata + assert not zrro.flags.writeable + + zc[1, 2] = 99 + expect = np.array([[11.0, 12, 13], [21, 22, 99], [31, 32, 33]]) + # We should have just changed zc, of course, but also zcro and the original eigen matrix + assert np.all(zc == expect) + assert np.all(zcro == expect) + assert np.all(m.get_cm_ref() == expect) + + zr[1, 2] = 99 + assert np.all(zr == expect) + assert np.all(zrro == expect) + assert np.all(m.get_rm_ref() == expect) + + # Make sure the readonly ones are numpy-readonly: + with pytest.raises(ValueError): + zcro[1, 2] = 6 + with pytest.raises(ValueError): + zrro[1, 2] = 6 + + # We should be able to explicitly copy like this (and since we're copying, + # the const should drop away) + y1 = np.array(m.get_cm_const_ref()) + + assert y1.flags.owndata + assert y1.flags.writeable + # We should get copies of the eigen data, which was modified above: + assert y1[1, 2] == 99 + y1[1, 2] += 12 + assert y1[1, 2] == 111 + assert zc[1, 2] == 99 # Make sure we aren't referencing the original + + +def test_both_ref_mutators(): + """Tests a complex chain of nested eigen/numpy references""" + + m.reset_refs() # In case another test already changed it + + z = m.get_cm_ref() # numpy -> eigen + z[0, 2] -= 3 + z2 = m.incr_matrix(z, 1) # numpy -> eigen -> numpy -> eigen + z2[1, 1] += 6 + z3 = m.incr_matrix(z, 2) # (numpy -> eigen)^3 + z3[2, 2] += -5 + z4 = m.incr_matrix(z, 3) # (numpy -> eigen)^4 + z4[1, 1] -= 1 + z5 = m.incr_matrix(z, 4) # (numpy -> eigen)^5 + z5[0, 0] = 0 + assert np.all(z == z2) + assert np.all(z == z3) + assert np.all(z == z4) + assert np.all(z == z5) + expect = np.array([[0.0, 22, 20], [31, 37, 33], [41, 42, 38]]) + assert np.all(z == expect) + + y = np.array(range(100), dtype="float64").reshape(10, 10) + y2 = m.incr_matrix_any(y, 10) # np -> eigen -> np + y3 = m.incr_matrix_any( + y2[0::2, 0::2], -33 + ) # np -> eigen -> np slice -> np -> eigen -> np + y4 = m.even_rows(y3) # numpy -> eigen slice -> (... y3) + y5 = m.even_cols(y4) # numpy -> eigen slice -> (... y4) + y6 = m.incr_matrix_any(y5, 1000) # numpy -> eigen -> (... y5) + + # Apply same mutations using just numpy: + yexpect = np.array(range(100), dtype="float64").reshape(10, 10) + yexpect += 10 + yexpect[0::2, 0::2] -= 33 + yexpect[0::4, 0::4] += 1000 + assert np.all(y6 == yexpect[0::4, 0::4]) + assert np.all(y5 == yexpect[0::4, 0::4]) + assert np.all(y4 == yexpect[0::4, 0::2]) + assert np.all(y3 == yexpect[0::2, 0::2]) + assert np.all(y2 == yexpect) + assert np.all(y == yexpect) + + +def test_nocopy_wrapper(): + # get_elem requires a column-contiguous matrix reference, but should be + # callable with other types of matrix (via copying): + int_matrix_colmajor = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]], order="F") + dbl_matrix_colmajor = np.array( + int_matrix_colmajor, dtype="double", order="F", copy=True + ) + int_matrix_rowmajor = np.array(int_matrix_colmajor, order="C", copy=True) + dbl_matrix_rowmajor = np.array( + int_matrix_rowmajor, dtype="double", order="C", copy=True + ) + + # All should be callable via get_elem: + assert m.get_elem(int_matrix_colmajor) == 8 + assert m.get_elem(dbl_matrix_colmajor) == 8 + assert m.get_elem(int_matrix_rowmajor) == 8 + assert m.get_elem(dbl_matrix_rowmajor) == 8 + + # All but the second should fail with m.get_elem_nocopy: + with pytest.raises(TypeError) as excinfo: + m.get_elem_nocopy(int_matrix_colmajor) + assert "get_elem_nocopy(): incompatible function arguments." in str(excinfo.value) + assert ", flags.f_contiguous" in str(excinfo.value) + assert m.get_elem_nocopy(dbl_matrix_colmajor) == 8 + with pytest.raises(TypeError) as excinfo: + m.get_elem_nocopy(int_matrix_rowmajor) + assert "get_elem_nocopy(): incompatible function arguments." in str(excinfo.value) + assert ", flags.f_contiguous" in str(excinfo.value) + with pytest.raises(TypeError) as excinfo: + m.get_elem_nocopy(dbl_matrix_rowmajor) + assert "get_elem_nocopy(): incompatible function arguments." in str(excinfo.value) + assert ", flags.f_contiguous" in str(excinfo.value) + + # For the row-major test, we take a long matrix in row-major, so only the third is allowed: + with pytest.raises(TypeError) as excinfo: + m.get_elem_rm_nocopy(int_matrix_colmajor) + assert "get_elem_rm_nocopy(): incompatible function arguments." in str( + excinfo.value + ) + assert ", flags.c_contiguous" in str(excinfo.value) + with pytest.raises(TypeError) as excinfo: + m.get_elem_rm_nocopy(dbl_matrix_colmajor) + assert "get_elem_rm_nocopy(): incompatible function arguments." in str( + excinfo.value + ) + assert ", flags.c_contiguous" in str(excinfo.value) + assert m.get_elem_rm_nocopy(int_matrix_rowmajor) == 8 + with pytest.raises(TypeError) as excinfo: + m.get_elem_rm_nocopy(dbl_matrix_rowmajor) + assert "get_elem_rm_nocopy(): incompatible function arguments." in str( + excinfo.value + ) + assert ", flags.c_contiguous" in str(excinfo.value) + + +def test_eigen_ref_life_support(): + """Ensure the lifetime of temporary arrays created by the `Ref` caster + + The `Ref` caster sometimes creates a copy which needs to stay alive. This needs to + happen both for directs casts (just the array) or indirectly (e.g. list of arrays). + """ + + a = np.full(shape=10, fill_value=8, dtype=np.int8) + assert m.get_elem_direct(a) == 8 + + list_of_a = [a] + assert m.get_elem_indirect(list_of_a) == 8 + + +def test_special_matrix_objects(): + assert np.all(m.incr_diag(7) == np.diag([1.0, 2, 3, 4, 5, 6, 7])) + + asymm = np.array([[1.0, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12], [13, 14, 15, 16]]) + symm_lower = np.array(asymm) + symm_upper = np.array(asymm) + for i in range(4): + for j in range(i + 1, 4): + symm_lower[i, j] = symm_lower[j, i] + symm_upper[j, i] = symm_upper[i, j] + + assert np.all(m.symmetric_lower(asymm) == symm_lower) + assert np.all(m.symmetric_upper(asymm) == symm_upper) + + +def test_dense_signature(doc): + assert ( + doc(m.double_col) + == """ + double_col(arg0: numpy.ndarray[numpy.float32[m, 1]]) -> numpy.ndarray[numpy.float32[m, 1]] + """ + ) + assert ( + doc(m.double_row) + == """ + double_row(arg0: numpy.ndarray[numpy.float32[1, n]]) -> numpy.ndarray[numpy.float32[1, n]] + """ + ) + assert doc(m.double_complex) == ( + """ + double_complex(arg0: numpy.ndarray[numpy.complex64[m, 1]])""" + """ -> numpy.ndarray[numpy.complex64[m, 1]] + """ + ) + assert doc(m.double_mat_rm) == ( + """ + double_mat_rm(arg0: numpy.ndarray[numpy.float32[m, n]])""" + """ -> numpy.ndarray[numpy.float32[m, n]] + """ + ) + + +def test_named_arguments(): + a = np.array([[1.0, 2], [3, 4], [5, 6]]) + b = np.ones((2, 1)) + + assert np.all(m.matrix_multiply(a, b) == np.array([[3.0], [7], [11]])) + assert np.all(m.matrix_multiply(A=a, B=b) == np.array([[3.0], [7], [11]])) + assert np.all(m.matrix_multiply(B=b, A=a) == np.array([[3.0], [7], [11]])) + + with pytest.raises(ValueError) as excinfo: + m.matrix_multiply(b, a) + assert str(excinfo.value) == "Nonconformable matrices!" + + with pytest.raises(ValueError) as excinfo: + m.matrix_multiply(A=b, B=a) + assert str(excinfo.value) == "Nonconformable matrices!" + + with pytest.raises(ValueError) as excinfo: + m.matrix_multiply(B=a, A=b) + assert str(excinfo.value) == "Nonconformable matrices!" + + +def test_sparse(): + pytest.importorskip("scipy") + assert_sparse_equal_ref(m.sparse_r()) + assert_sparse_equal_ref(m.sparse_c()) + assert_sparse_equal_ref(m.sparse_copy_r(m.sparse_r())) + assert_sparse_equal_ref(m.sparse_copy_c(m.sparse_c())) + assert_sparse_equal_ref(m.sparse_copy_r(m.sparse_c())) + assert_sparse_equal_ref(m.sparse_copy_c(m.sparse_r())) + + +def test_sparse_signature(doc): + pytest.importorskip("scipy") + assert ( + doc(m.sparse_copy_r) + == """ + sparse_copy_r(arg0: scipy.sparse.csr_matrix[numpy.float32]) -> scipy.sparse.csr_matrix[numpy.float32] + """ + ) + assert ( + doc(m.sparse_copy_c) + == """ + sparse_copy_c(arg0: scipy.sparse.csc_matrix[numpy.float32]) -> scipy.sparse.csc_matrix[numpy.float32] + """ + ) + + +def test_issue738(): + """Ignore strides on a length-1 dimension (even if they would be incompatible length > 1)""" + assert np.all(m.iss738_f1(np.array([[1.0, 2, 3]])) == np.array([[1.0, 102, 203]])) + assert np.all( + m.iss738_f1(np.array([[1.0], [2], [3]])) == np.array([[1.0], [12], [23]]) + ) + + assert np.all(m.iss738_f2(np.array([[1.0, 2, 3]])) == np.array([[1.0, 102, 203]])) + assert np.all( + m.iss738_f2(np.array([[1.0], [2], [3]])) == np.array([[1.0], [12], [23]]) + ) + + +@pytest.mark.parametrize("func", [m.iss738_f1, m.iss738_f2]) +@pytest.mark.parametrize("sizes", [(0, 2), (2, 0)]) +def test_zero_length(func, sizes): + """Ignore strides on a length-0 dimension (even if they would be incompatible length > 1)""" + assert np.all(func(np.zeros(sizes)) == np.zeros(sizes)) + + +def test_issue1105(): + """Issue 1105: 1xN or Nx1 input arrays weren't accepted for eigen + compile-time row vectors or column vector""" + assert m.iss1105_row(np.ones((1, 7))) + assert m.iss1105_col(np.ones((7, 1))) + + # These should still fail (incompatible dimensions): + with pytest.raises(TypeError) as excinfo: + m.iss1105_row(np.ones((7, 1))) + assert "incompatible function arguments" in str(excinfo.value) + with pytest.raises(TypeError) as excinfo: + m.iss1105_col(np.ones((1, 7))) + assert "incompatible function arguments" in str(excinfo.value) + + +def test_custom_operator_new(): + """Using Eigen types as member variables requires a class-specific + operator new with proper alignment""" + + o = m.CustomOperatorNew() + np.testing.assert_allclose(o.a, 0.0) + np.testing.assert_allclose(o.b.diagonal(), 1.0) diff --git a/pybind11/tests/test_eigen_tensor.cpp b/pybind11/tests/test_eigen_tensor.cpp new file mode 100644 index 0000000..503c69c --- /dev/null +++ b/pybind11/tests/test_eigen_tensor.cpp @@ -0,0 +1,18 @@ +/* + tests/eigen_tensor.cpp -- automatic conversion of Eigen Tensor + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#define PYBIND11_TEST_EIGEN_TENSOR_NAMESPACE eigen_tensor + +#ifdef EIGEN_AVOID_STL_ARRAY +# undef EIGEN_AVOID_STL_ARRAY +#endif + +#include "test_eigen_tensor.inl" + +#include "pybind11_tests.h" + +test_initializer egien_tensor("eigen_tensor", eigen_tensor_test::test_module); diff --git a/pybind11/tests/test_eigen_tensor.inl b/pybind11/tests/test_eigen_tensor.inl new file mode 100644 index 0000000..d864ce7 --- /dev/null +++ b/pybind11/tests/test_eigen_tensor.inl @@ -0,0 +1,333 @@ +/* + tests/eigen_tensor.cpp -- automatic conversion of Eigen Tensor + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#include + +PYBIND11_NAMESPACE_BEGIN(eigen_tensor_test) + +namespace py = pybind11; + +PYBIND11_WARNING_DISABLE_MSVC(4127) + +template +void reset_tensor(M &x) { + for (int i = 0; i < x.dimension(0); i++) { + for (int j = 0; j < x.dimension(1); j++) { + for (int k = 0; k < x.dimension(2); k++) { + x(i, j, k) = i * (5 * 2) + j * 2 + k; + } + } + } +} + +template +bool check_tensor(M &x) { + for (int i = 0; i < x.dimension(0); i++) { + for (int j = 0; j < x.dimension(1); j++) { + for (int k = 0; k < x.dimension(2); k++) { + if (x(i, j, k) != (i * (5 * 2) + j * 2 + k)) { + return false; + } + } + } + } + return true; +} + +template +Eigen::Tensor &get_tensor() { + static Eigen::Tensor *x; + + if (!x) { + x = new Eigen::Tensor(3, 5, 2); + reset_tensor(*x); + } + + return *x; +} + +template +Eigen::TensorMap> &get_tensor_map() { + static Eigen::TensorMap> *x; + + if (!x) { + x = new Eigen::TensorMap>(get_tensor()); + } + + return *x; +} + +template +Eigen::TensorFixedSize, Options> &get_fixed_tensor() { + static Eigen::TensorFixedSize, Options> *x; + + if (!x) { + Eigen::aligned_allocator, Options>> + allocator; + x = new (allocator.allocate(1)) + Eigen::TensorFixedSize, Options>(); + reset_tensor(*x); + } + + return *x; +} + +template +const Eigen::Tensor &get_const_tensor() { + return get_tensor(); +} + +template +struct CustomExample { + CustomExample() : member(get_tensor()), view_member(member) {} + + Eigen::Tensor member; + Eigen::TensorMap> view_member; +}; + +template +void init_tensor_module(pybind11::module &m) { + const char *needed_options = ""; + if (Options == Eigen::ColMajor) { + needed_options = "F"; + } else { + needed_options = "C"; + } + m.attr("needed_options") = needed_options; + + m.def("setup", []() { + reset_tensor(get_tensor()); + reset_tensor(get_fixed_tensor()); + }); + + m.def("is_ok", []() { + return check_tensor(get_tensor()) && check_tensor(get_fixed_tensor()); + }); + + py::class_>(m, "CustomExample", py::module_local()) + .def(py::init<>()) + .def_readonly( + "member", &CustomExample::member, py::return_value_policy::reference_internal) + .def_readonly("member_view", + &CustomExample::view_member, + py::return_value_policy::reference_internal); + + m.def( + "copy_fixed_tensor", + []() { return &get_fixed_tensor(); }, + py::return_value_policy::copy); + + m.def( + "copy_tensor", []() { return &get_tensor(); }, py::return_value_policy::copy); + + m.def( + "copy_const_tensor", + []() { return &get_const_tensor(); }, + py::return_value_policy::copy); + + m.def( + "move_fixed_tensor_copy", + []() -> Eigen::TensorFixedSize, Options> { + return get_fixed_tensor(); + }, + py::return_value_policy::move); + + m.def( + "move_tensor_copy", + []() -> Eigen::Tensor { return get_tensor(); }, + py::return_value_policy::move); + + m.def( + "move_const_tensor", + []() -> const Eigen::Tensor & { return get_const_tensor(); }, + py::return_value_policy::move); + + m.def( + "take_fixed_tensor", + + []() { + Eigen::aligned_allocator< + Eigen::TensorFixedSize, Options>> + allocator; + return new (allocator.allocate(1)) + Eigen::TensorFixedSize, Options>( + get_fixed_tensor()); + }, + py::return_value_policy::take_ownership); + + m.def( + "take_tensor", + []() { return new Eigen::Tensor(get_tensor()); }, + py::return_value_policy::take_ownership); + + m.def( + "take_const_tensor", + []() -> const Eigen::Tensor * { + return new Eigen::Tensor(get_tensor()); + }, + py::return_value_policy::take_ownership); + + m.def( + "take_view_tensor", + []() -> const Eigen::TensorMap> * { + return new Eigen::TensorMap>(get_tensor()); + }, + py::return_value_policy::take_ownership); + + m.def( + "reference_tensor", + []() { return &get_tensor(); }, + py::return_value_policy::reference); + + m.def( + "reference_tensor_v2", + []() -> Eigen::Tensor & { return get_tensor(); }, + py::return_value_policy::reference); + + m.def( + "reference_tensor_internal", + []() { return &get_tensor(); }, + py::return_value_policy::reference_internal); + + m.def( + "reference_fixed_tensor", + []() { return &get_tensor(); }, + py::return_value_policy::reference); + + m.def( + "reference_const_tensor", + []() { return &get_const_tensor(); }, + py::return_value_policy::reference); + + m.def( + "reference_const_tensor_v2", + []() -> const Eigen::Tensor & { return get_const_tensor(); }, + py::return_value_policy::reference); + + m.def( + "reference_view_of_tensor", + []() -> Eigen::TensorMap> { + return get_tensor_map(); + }, + py::return_value_policy::reference); + + m.def( + "reference_view_of_tensor_v2", + // NOLINTNEXTLINE(readability-const-return-type) + []() -> const Eigen::TensorMap> { + return get_tensor_map(); // NOLINT(readability-const-return-type) + }, // NOLINT(readability-const-return-type) + py::return_value_policy::reference); + + m.def( + "reference_view_of_tensor_v3", + []() -> Eigen::TensorMap> * { + return &get_tensor_map(); + }, + py::return_value_policy::reference); + + m.def( + "reference_view_of_tensor_v4", + []() -> const Eigen::TensorMap> * { + return &get_tensor_map(); + }, + py::return_value_policy::reference); + + m.def( + "reference_view_of_tensor_v5", + []() -> Eigen::TensorMap> & { + return get_tensor_map(); + }, + py::return_value_policy::reference); + + m.def( + "reference_view_of_tensor_v6", + []() -> const Eigen::TensorMap> & { + return get_tensor_map(); + }, + py::return_value_policy::reference); + + m.def( + "reference_view_of_fixed_tensor", + []() { + return Eigen::TensorMap< + Eigen::TensorFixedSize, Options>>( + get_fixed_tensor()); + }, + py::return_value_policy::reference); + + m.def("round_trip_tensor", + [](const Eigen::Tensor &tensor) { return tensor; }); + + m.def( + "round_trip_tensor_noconvert", + [](const Eigen::Tensor &tensor) { return tensor; }, + py::arg("tensor").noconvert()); + + m.def("round_trip_tensor2", + [](const Eigen::Tensor &tensor) { return tensor; }); + + m.def("round_trip_fixed_tensor", + [](const Eigen::TensorFixedSize, Options> &tensor) { + return tensor; + }); + + m.def( + "round_trip_view_tensor", + [](Eigen::TensorMap> view) { return view; }, + py::return_value_policy::reference); + + m.def( + "round_trip_view_tensor_ref", + [](Eigen::TensorMap> &view) { return view; }, + py::return_value_policy::reference); + + m.def( + "round_trip_view_tensor_ptr", + [](Eigen::TensorMap> *view) { return view; }, + py::return_value_policy::reference); + + m.def( + "round_trip_aligned_view_tensor", + [](Eigen::TensorMap, Eigen::Aligned> view) { + return view; + }, + py::return_value_policy::reference); + + m.def( + "round_trip_const_view_tensor", + [](Eigen::TensorMap> view) { + return Eigen::Tensor(view); + }, + py::return_value_policy::move); + + m.def( + "round_trip_rank_0", + [](const Eigen::Tensor &tensor) { return tensor; }, + py::return_value_policy::move); + + m.def( + "round_trip_rank_0_noconvert", + [](const Eigen::Tensor &tensor) { return tensor; }, + py::arg("tensor").noconvert(), + py::return_value_policy::move); + + m.def( + "round_trip_rank_0_view", + [](Eigen::TensorMap> &tensor) { return tensor; }, + py::return_value_policy::reference); +} + +void test_module(py::module_ &m) { + auto f_style = m.def_submodule("f_style"); + auto c_style = m.def_submodule("c_style"); + + init_tensor_module(f_style); + init_tensor_module(c_style); +} + +PYBIND11_NAMESPACE_END(eigen_tensor_test) diff --git a/pybind11/tests/test_eigen_tensor.py b/pybind11/tests/test_eigen_tensor.py new file mode 100644 index 0000000..3e7ee6b --- /dev/null +++ b/pybind11/tests/test_eigen_tensor.py @@ -0,0 +1,288 @@ +import sys + +import pytest + +np = pytest.importorskip("numpy") +eigen_tensor = pytest.importorskip("pybind11_tests.eigen_tensor") +submodules = [eigen_tensor.c_style, eigen_tensor.f_style] +try: + import eigen_tensor_avoid_stl_array as avoid + + submodules += [avoid.c_style, avoid.f_style] +except ImportError as e: + # Ensure config, build, toolchain, etc. issues are not masked here: + msg = ( + "import eigen_tensor_avoid_stl_array FAILED, while " + "import pybind11_tests.eigen_tensor succeeded. " + "Please ensure that " + "test_eigen_tensor.cpp & " + "eigen_tensor_avoid_stl_array.cpp " + "are built together (or both are not built if Eigen is not available)." + ) + raise RuntimeError(msg) from e + +tensor_ref = np.empty((3, 5, 2), dtype=np.int64) + +for i in range(tensor_ref.shape[0]): + for j in range(tensor_ref.shape[1]): + for k in range(tensor_ref.shape[2]): + tensor_ref[i, j, k] = i * (5 * 2) + j * 2 + k + +indices = (2, 3, 1) + + +@pytest.fixture(autouse=True) +def cleanup(): + for module in submodules: + module.setup() + + yield + + for module in submodules: + assert module.is_ok() + + +def test_import_avoid_stl_array(): + pytest.importorskip("eigen_tensor_avoid_stl_array") + assert len(submodules) == 4 + + +def assert_equal_tensor_ref(mat, writeable=True, modified=None): + assert mat.flags.writeable == writeable + + copy = np.array(tensor_ref) + if modified is not None: + copy[indices] = modified + + np.testing.assert_array_equal(mat, copy) + + +@pytest.mark.parametrize("m", submodules) +@pytest.mark.parametrize("member_name", ["member", "member_view"]) +def test_reference_internal(m, member_name): + if not hasattr(sys, "getrefcount"): + pytest.skip("No reference counting") + foo = m.CustomExample() + counts = sys.getrefcount(foo) + mem = getattr(foo, member_name) + assert_equal_tensor_ref(mem, writeable=False) + new_counts = sys.getrefcount(foo) + assert new_counts == counts + 1 + assert_equal_tensor_ref(mem, writeable=False) + del mem + assert sys.getrefcount(foo) == counts + + +assert_equal_funcs = [ + "copy_tensor", + "copy_fixed_tensor", + "copy_const_tensor", + "move_tensor_copy", + "move_fixed_tensor_copy", + "take_tensor", + "take_fixed_tensor", + "reference_tensor", + "reference_tensor_v2", + "reference_fixed_tensor", + "reference_view_of_tensor", + "reference_view_of_tensor_v3", + "reference_view_of_tensor_v5", + "reference_view_of_fixed_tensor", +] + +assert_equal_const_funcs = [ + "reference_view_of_tensor_v2", + "reference_view_of_tensor_v4", + "reference_view_of_tensor_v6", + "reference_const_tensor", + "reference_const_tensor_v2", +] + + +@pytest.mark.parametrize("m", submodules) +@pytest.mark.parametrize("func_name", assert_equal_funcs + assert_equal_const_funcs) +def test_convert_tensor_to_py(m, func_name): + writeable = func_name in assert_equal_funcs + assert_equal_tensor_ref(getattr(m, func_name)(), writeable=writeable) + + +@pytest.mark.parametrize("m", submodules) +def test_bad_cpp_to_python_casts(m): + with pytest.raises( + RuntimeError, match="Cannot use reference internal when there is no parent" + ): + m.reference_tensor_internal() + + with pytest.raises(RuntimeError, match="Cannot move from a constant reference"): + m.move_const_tensor() + + with pytest.raises( + RuntimeError, match="Cannot take ownership of a const reference" + ): + m.take_const_tensor() + + with pytest.raises( + RuntimeError, + match="Invalid return_value_policy for Eigen Map type, must be either reference or reference_internal", + ): + m.take_view_tensor() + + +@pytest.mark.parametrize("m", submodules) +def test_bad_python_to_cpp_casts(m): + with pytest.raises( + TypeError, match=r"^round_trip_tensor\(\): incompatible function arguments" + ): + m.round_trip_tensor(np.zeros((2, 3))) + + with pytest.raises(TypeError, match=r"^Cannot cast array data from dtype"): + m.round_trip_tensor(np.zeros(dtype=np.str_, shape=(2, 3, 1))) + + with pytest.raises( + TypeError, + match=r"^round_trip_tensor_noconvert\(\): incompatible function arguments", + ): + m.round_trip_tensor_noconvert(tensor_ref) + + assert_equal_tensor_ref( + m.round_trip_tensor_noconvert(tensor_ref.astype(np.float64)) + ) + + bad_options = "C" if m.needed_options == "F" else "F" + # Shape, dtype and the order need to be correct for a TensorMap cast + with pytest.raises( + TypeError, match=r"^round_trip_view_tensor\(\): incompatible function arguments" + ): + m.round_trip_view_tensor( + np.zeros((3, 5, 2), dtype=np.float64, order=bad_options) + ) + + with pytest.raises( + TypeError, match=r"^round_trip_view_tensor\(\): incompatible function arguments" + ): + m.round_trip_view_tensor( + np.zeros((3, 5, 2), dtype=np.float32, order=m.needed_options) + ) + + with pytest.raises( + TypeError, match=r"^round_trip_view_tensor\(\): incompatible function arguments" + ): + m.round_trip_view_tensor( + np.zeros((3, 5), dtype=np.float64, order=m.needed_options) + ) + + temp = np.zeros((3, 5, 2), dtype=np.float64, order=m.needed_options) + with pytest.raises( + TypeError, match=r"^round_trip_view_tensor\(\): incompatible function arguments" + ): + m.round_trip_view_tensor( + temp[:, ::-1, :], + ) + + temp = np.zeros((3, 5, 2), dtype=np.float64, order=m.needed_options) + temp.setflags(write=False) + with pytest.raises( + TypeError, match=r"^round_trip_view_tensor\(\): incompatible function arguments" + ): + m.round_trip_view_tensor(temp) + + +@pytest.mark.parametrize("m", submodules) +def test_references_actually_refer(m): + a = m.reference_tensor() + temp = a[indices] + a[indices] = 100 + assert_equal_tensor_ref(m.copy_const_tensor(), modified=100) + a[indices] = temp + assert_equal_tensor_ref(m.copy_const_tensor()) + + a = m.reference_view_of_tensor() + a[indices] = 100 + assert_equal_tensor_ref(m.copy_const_tensor(), modified=100) + a[indices] = temp + assert_equal_tensor_ref(m.copy_const_tensor()) + + +@pytest.mark.parametrize("m", submodules) +def test_round_trip(m): + assert_equal_tensor_ref(m.round_trip_tensor(tensor_ref)) + + with pytest.raises(TypeError, match="^Cannot cast array data from"): + assert_equal_tensor_ref(m.round_trip_tensor2(tensor_ref)) + + assert_equal_tensor_ref(m.round_trip_tensor2(np.array(tensor_ref, dtype=np.int32))) + assert_equal_tensor_ref(m.round_trip_fixed_tensor(tensor_ref)) + assert_equal_tensor_ref(m.round_trip_aligned_view_tensor(m.reference_tensor())) + + copy = np.array(tensor_ref, dtype=np.float64, order=m.needed_options) + assert_equal_tensor_ref(m.round_trip_view_tensor(copy)) + assert_equal_tensor_ref(m.round_trip_view_tensor_ref(copy)) + assert_equal_tensor_ref(m.round_trip_view_tensor_ptr(copy)) + copy.setflags(write=False) + assert_equal_tensor_ref(m.round_trip_const_view_tensor(copy)) + + np.testing.assert_array_equal( + tensor_ref[:, ::-1, :], m.round_trip_tensor(tensor_ref[:, ::-1, :]) + ) + + assert m.round_trip_rank_0(np.float64(3.5)) == 3.5 + assert m.round_trip_rank_0(3.5) == 3.5 + + with pytest.raises( + TypeError, + match=r"^round_trip_rank_0_noconvert\(\): incompatible function arguments", + ): + m.round_trip_rank_0_noconvert(np.float64(3.5)) + + with pytest.raises( + TypeError, + match=r"^round_trip_rank_0_noconvert\(\): incompatible function arguments", + ): + m.round_trip_rank_0_noconvert(3.5) + + with pytest.raises( + TypeError, match=r"^round_trip_rank_0_view\(\): incompatible function arguments" + ): + m.round_trip_rank_0_view(np.float64(3.5)) + + with pytest.raises( + TypeError, match=r"^round_trip_rank_0_view\(\): incompatible function arguments" + ): + m.round_trip_rank_0_view(3.5) + + +@pytest.mark.parametrize("m", submodules) +def test_round_trip_references_actually_refer(m): + # Need to create a copy that matches the type on the C side + copy = np.array(tensor_ref, dtype=np.float64, order=m.needed_options) + a = m.round_trip_view_tensor(copy) + temp = a[indices] + a[indices] = 100 + assert_equal_tensor_ref(copy, modified=100) + a[indices] = temp + assert_equal_tensor_ref(copy) + + +@pytest.mark.parametrize("m", submodules) +def test_doc_string(m, doc): + assert ( + doc(m.copy_tensor) == "copy_tensor() -> numpy.ndarray[numpy.float64[?, ?, ?]]" + ) + assert ( + doc(m.copy_fixed_tensor) + == "copy_fixed_tensor() -> numpy.ndarray[numpy.float64[3, 5, 2]]" + ) + assert ( + doc(m.reference_const_tensor) + == "reference_const_tensor() -> numpy.ndarray[numpy.float64[?, ?, ?]]" + ) + + order_flag = f"flags.{m.needed_options.lower()}_contiguous" + assert doc(m.round_trip_view_tensor) == ( + f"round_trip_view_tensor(arg0: numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, {order_flag}])" + f" -> numpy.ndarray[numpy.float64[?, ?, ?], flags.writeable, {order_flag}]" + ) + assert doc(m.round_trip_const_view_tensor) == ( + f"round_trip_const_view_tensor(arg0: numpy.ndarray[numpy.float64[?, ?, ?], {order_flag}])" + " -> numpy.ndarray[numpy.float64[?, ?, ?]]" + ) diff --git a/pybind11/tests/test_embed/catch.cpp b/pybind11/tests/test_embed/catch.cpp index 96d2e3f..558a7a3 100644 --- a/pybind11/tests/test_embed/catch.cpp +++ b/pybind11/tests/test_embed/catch.cpp @@ -3,11 +3,9 @@ #include -#ifdef _MSC_VER // Silence MSVC C++17 deprecation warning from Catch regarding std::uncaught_exceptions (up to // catch 2.0.1; this should be fixed in the next catch release after 2.0.1). -# pragma warning(disable : 4996) -#endif +PYBIND11_WARNING_DISABLE_MSVC(4996) // Catch uses _ internally, which breaks gettext style defines #ifdef _ @@ -20,7 +18,25 @@ namespace py = pybind11; int main(int argc, char *argv[]) { + // Setup for TEST_CASE in test_interpreter.cpp, tagging on a large random number: + std::string updated_pythonpath("pybind11_test_embed_PYTHONPATH_2099743835476552"); + const char *preexisting_pythonpath = getenv("PYTHONPATH"); + if (preexisting_pythonpath != nullptr) { +#if defined(_WIN32) + updated_pythonpath += ';'; +#else + updated_pythonpath += ':'; +#endif + updated_pythonpath += preexisting_pythonpath; + } +#if defined(_WIN32) + _putenv_s("PYTHONPATH", updated_pythonpath.c_str()); +#else + setenv("PYTHONPATH", updated_pythonpath.c_str(), /*replace=*/1); +#endif + py::scoped_interpreter guard{}; + auto result = Catch::Session().run(argc, argv); return result < 0xff ? result : 0xff; diff --git a/pybind11/tests/test_embed/test_interpreter.cpp b/pybind11/tests/test_embed/test_interpreter.cpp index 1c45457..c6c8a22 100644 --- a/pybind11/tests/test_embed/test_interpreter.cpp +++ b/pybind11/tests/test_embed/test_interpreter.cpp @@ -1,10 +1,8 @@ #include -#ifdef _MSC_VER // Silence MSVC C++17 deprecation warning from Catch regarding std::uncaught_exceptions (up to // catch 2.0.1; this should be fixed in the next catch release after 2.0.1). -# pragma warning(disable : 4996) -#endif +PYBIND11_WARNING_DISABLE_MSVC(4996) #include #include @@ -16,6 +14,11 @@ namespace py = pybind11; using namespace py::literals; +size_t get_sys_path_size() { + auto sys_path = py::module::import("sys").attr("path"); + return py::len(sys_path); +} + class Widget { public: explicit Widget(std::string message) : message(std::move(message)) {} @@ -75,6 +78,13 @@ PYBIND11_EMBEDDED_MODULE(throw_error_already_set, ) { d["missing"].cast(); } +TEST_CASE("PYTHONPATH is used to update sys.path") { + // The setup for this TEST_CASE is in catch.cpp! + auto sys_path = py::str(py::module_::import("sys").attr("path")).cast(); + REQUIRE_THAT(sys_path, + Catch::Matchers::Contains("pybind11_test_embed_PYTHONPATH_2099743835476552")); +} + TEST_CASE("Pass classes and data between modules defined in C++ and Python") { auto module_ = py::module_::import("test_interpreter"); REQUIRE(py::hasattr(module_, "DerivedWidget")); @@ -161,10 +171,94 @@ TEST_CASE("There can be only one interpreter") { py::initialize_interpreter(); } -bool has_pybind11_internals_builtin() { - auto builtins = py::handle(PyEval_GetBuiltins()); - return builtins.contains(PYBIND11_INTERNALS_ID); -}; +#if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX +TEST_CASE("Custom PyConfig") { + py::finalize_interpreter(); + PyConfig config; + PyConfig_InitPythonConfig(&config); + REQUIRE_NOTHROW(py::scoped_interpreter{&config}); + { + py::scoped_interpreter p{&config}; + REQUIRE(py::module_::import("widget_module").attr("add")(1, 41).cast() == 42); + } + py::initialize_interpreter(); +} + +TEST_CASE("scoped_interpreter with PyConfig_InitIsolatedConfig and argv") { + py::finalize_interpreter(); + { + PyConfig config; + PyConfig_InitIsolatedConfig(&config); + char *argv[] = {strdup("a.out")}; + py::scoped_interpreter argv_scope{&config, 1, argv}; + std::free(argv[0]); + auto module = py::module::import("test_interpreter"); + auto py_widget = module.attr("DerivedWidget")("The question"); + const auto &cpp_widget = py_widget.cast(); + REQUIRE(cpp_widget.argv0() == "a.out"); + } + py::initialize_interpreter(); +} + +TEST_CASE("scoped_interpreter with PyConfig_InitPythonConfig and argv") { + py::finalize_interpreter(); + { + PyConfig config; + PyConfig_InitPythonConfig(&config); + + // `initialize_interpreter() overrides the default value for config.parse_argv (`1`) by + // changing it to `0`. This test exercises `scoped_interpreter` with the default config. + char *argv[] = {strdup("a.out"), strdup("arg1")}; + py::scoped_interpreter argv_scope(&config, 2, argv); + std::free(argv[0]); + std::free(argv[1]); + auto module = py::module::import("test_interpreter"); + auto py_widget = module.attr("DerivedWidget")("The question"); + const auto &cpp_widget = py_widget.cast(); + REQUIRE(cpp_widget.argv0() == "arg1"); + } + py::initialize_interpreter(); +} +#endif + +TEST_CASE("Add program dir to path pre-PyConfig") { + py::finalize_interpreter(); + size_t path_size_add_program_dir_to_path_false = 0; + { + py::scoped_interpreter scoped_interp{true, 0, nullptr, false}; + path_size_add_program_dir_to_path_false = get_sys_path_size(); + } + { + py::scoped_interpreter scoped_interp{}; + REQUIRE(get_sys_path_size() == path_size_add_program_dir_to_path_false + 1); + } + py::initialize_interpreter(); +} + +#if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX +TEST_CASE("Add program dir to path using PyConfig") { + py::finalize_interpreter(); + size_t path_size_add_program_dir_to_path_false = 0; + { + PyConfig config; + PyConfig_InitPythonConfig(&config); + py::scoped_interpreter scoped_interp{&config, 0, nullptr, false}; + path_size_add_program_dir_to_path_false = get_sys_path_size(); + } + { + PyConfig config; + PyConfig_InitPythonConfig(&config); + py::scoped_interpreter scoped_interp{&config}; + REQUIRE(get_sys_path_size() == path_size_add_program_dir_to_path_false + 1); + } + py::initialize_interpreter(); +} +#endif + +bool has_state_dict_internals_obj() { + return bool( + py::detail::get_internals_obj_from_state_dict(py::detail::get_python_state_dict())); +} bool has_pybind11_internals_static() { auto **&ipp = py::detail::get_internals_pp(); @@ -174,7 +268,7 @@ bool has_pybind11_internals_static() { TEST_CASE("Restart the interpreter") { // Verify pre-restart state. REQUIRE(py::module_::import("widget_module").attr("add")(1, 2).cast() == 3); - REQUIRE(has_pybind11_internals_builtin()); + REQUIRE(has_state_dict_internals_obj()); REQUIRE(has_pybind11_internals_static()); REQUIRE(py::module_::import("external_module").attr("A")(123).attr("value").cast() == 123); @@ -191,10 +285,10 @@ TEST_CASE("Restart the interpreter") { REQUIRE(Py_IsInitialized() == 1); // Internals are deleted after a restart. - REQUIRE_FALSE(has_pybind11_internals_builtin()); + REQUIRE_FALSE(has_state_dict_internals_obj()); REQUIRE_FALSE(has_pybind11_internals_static()); pybind11::detail::get_internals(); - REQUIRE(has_pybind11_internals_builtin()); + REQUIRE(has_state_dict_internals_obj()); REQUIRE(has_pybind11_internals_static()); REQUIRE(reinterpret_cast(*py::detail::get_internals_pp()) == py::module_::import("external_module").attr("internals_at")().cast()); @@ -209,13 +303,13 @@ TEST_CASE("Restart the interpreter") { py::detail::get_internals(); *static_cast(ran) = true; }); - REQUIRE_FALSE(has_pybind11_internals_builtin()); + REQUIRE_FALSE(has_state_dict_internals_obj()); REQUIRE_FALSE(has_pybind11_internals_static()); REQUIRE_FALSE(ran); py::finalize_interpreter(); REQUIRE(ran); py::initialize_interpreter(); - REQUIRE_FALSE(has_pybind11_internals_builtin()); + REQUIRE_FALSE(has_state_dict_internals_obj()); REQUIRE_FALSE(has_pybind11_internals_static()); // C++ modules can be reloaded. @@ -237,7 +331,7 @@ TEST_CASE("Subinterpreter") { REQUIRE(m.attr("add")(1, 2).cast() == 3); } - REQUIRE(has_pybind11_internals_builtin()); + REQUIRE(has_state_dict_internals_obj()); REQUIRE(has_pybind11_internals_static()); /// Create and switch to a subinterpreter. @@ -247,7 +341,7 @@ TEST_CASE("Subinterpreter") { // Subinterpreters get their own copy of builtins. detail::get_internals() still // works by returning from the static variable, i.e. all interpreters share a single // global pybind11::internals; - REQUIRE_FALSE(has_pybind11_internals_builtin()); + REQUIRE_FALSE(has_state_dict_internals_obj()); REQUIRE(has_pybind11_internals_static()); // Modules tags should be gone. @@ -286,7 +380,6 @@ TEST_CASE("Threads") { { py::gil_scoped_release gil_release{}; - REQUIRE(has_pybind11_internals_static()); auto threads = std::vector(); for (auto i = 0; i < num_threads; ++i) { diff --git a/pybind11/tests/test_enum.py b/pybind11/tests/test_enum.py index f14a723..4e85d29 100644 --- a/pybind11/tests/test_enum.py +++ b/pybind11/tests/test_enum.py @@ -1,3 +1,5 @@ +# ruff: noqa: SIM201 SIM300 SIM202 + import pytest from pybind11_tests import enums as m diff --git a/pybind11/tests/test_exceptions.cpp b/pybind11/tests/test_exceptions.cpp index 3ec999d..854c7e6 100644 --- a/pybind11/tests/test_exceptions.cpp +++ b/pybind11/tests/test_exceptions.cpp @@ -105,11 +105,6 @@ struct PythonAlreadySetInDestructor { py::str s; }; -std::string error_already_set_what(const py::object &exc_type, const py::object &exc_value) { - PyErr_SetObject(exc_type.ptr(), exc_value.ptr()); - return py::error_already_set().what(); -} - TEST_SUBMODULE(exceptions, m) { m.def("throw_std_exception", []() { throw std::runtime_error("This exception was intentionally thrown."); }); @@ -334,4 +329,19 @@ TEST_SUBMODULE(exceptions, m) { e.restore(); } }); + + // https://github.com/pybind/pybind11/issues/4075 + m.def("test_pypy_oserror_normalization", []() { + try { + py::module_::import("io").attr("open")("this_filename_must_not_exist", "r"); + } catch (const py::error_already_set &e) { + return py::str(e.what()); // str must be built before e goes out of scope. + } + return py::str("UNEXPECTED"); + }); + + m.def("test_fn_cast_int", [](const py::function &fn) { + // function returns None instead of int, should give a useful error message + fn().cast(); + }); } diff --git a/pybind11/tests/test_exceptions.py b/pybind11/tests/test_exceptions.py index a5984a1..ccac453 100644 --- a/pybind11/tests/test_exceptions.py +++ b/pybind11/tests/test_exceptions.py @@ -4,6 +4,7 @@ import env import pybind11_cross_module_tests as cm +import pybind11_tests # noqa: F401 from pybind11_tests import exceptions as m @@ -72,9 +73,9 @@ def test_cross_module_exceptions(msg): # TODO: FIXME @pytest.mark.xfail( - "env.PYPY and env.MACOS", + "env.MACOS and (env.PYPY or pybind11_tests.compiler_info.startswith('Homebrew Clang'))", raises=RuntimeError, - reason="Expected failure with PyPy and libc++ (Issue #2847 & PR #2999)", + reason="See Issue #2847, PR #2999, PR #4324", ) def test_cross_module_exception_translator(): with pytest.raises(KeyError): @@ -93,8 +94,7 @@ def ignore_pytest_unraisable_warning(f): if hasattr(pytest, unraisable): # Python >= 3.8 and pytest >= 6 dec = pytest.mark.filterwarnings(f"ignore::pytest.{unraisable}") return dec(f) - else: - return f + return f # TODO: find out why this fails on PyPy, https://foss.heptapod.net/pypy/pypy/-/issues/3583 @@ -182,11 +182,11 @@ def test_custom(msg): m.throws5_1() assert msg(excinfo.value) == "MyException5 subclass" - with pytest.raises(m.MyException5) as excinfo: + with pytest.raises(m.MyException5) as excinfo: # noqa: PT012 try: m.throws5() - except m.MyException5_1: - raise RuntimeError("Exception error: caught child from parent") + except m.MyException5_1 as err: + raise RuntimeError("Exception error: caught child from parent") from err assert msg(excinfo.value) == "this is a helper-defined translated exception" @@ -211,7 +211,7 @@ def throw_myex5(): m.try_catch(m.MyException5, throw_myex) assert str(excinfo.value) == "nested error" - def pycatch(exctype, f, *args): + def pycatch(exctype, f, *args): # noqa: ARG001 try: f(*args) except m.MyException as e: @@ -275,6 +275,20 @@ def test_local_translator(msg): assert msg(excinfo.value) == "this mod" +def test_error_already_set_message_with_unicode_surrogate(): # Issue #4288 + assert m.error_already_set_what(RuntimeError, "\ud927") == ( + "RuntimeError: \\ud927", + False, + ) + + +def test_error_already_set_message_with_malformed_utf8(): + assert m.error_already_set_what(RuntimeError, b"\x80") == ( + "RuntimeError: b'\\x80'", + False, + ) + + class FlakyException(Exception): def __init__(self, failure_point): if failure_point == "failure_point_init": @@ -288,12 +302,12 @@ def __str__(self): @pytest.mark.parametrize( - "exc_type, exc_value, expected_what", - ( + ("exc_type", "exc_value", "expected_what"), + [ (ValueError, "plain_str", "ValueError: plain_str"), (ValueError, ("tuple_elem",), "ValueError: tuple_elem"), (FlakyException, ("happy",), "FlakyException: FlakyException.__str__"), - ), + ], ) def test_error_already_set_what_with_happy_exceptions( exc_type, exc_value, expected_what @@ -303,8 +317,7 @@ def test_error_already_set_what_with_happy_exceptions( assert what == expected_what -@pytest.mark.skipif("env.PYPY", reason="PyErr_NormalizeException Segmentation fault") -def test_flaky_exception_failure_point_init(): +def _test_flaky_exception_failure_point_init_before_py_3_12(): with pytest.raises(RuntimeError) as excinfo: m.error_already_set_what(FlakyException, ("failure_point_init",)) lines = str(excinfo.value).splitlines() @@ -318,7 +331,33 @@ def test_flaky_exception_failure_point_init(): # Checking the first two lines of the traceback as formatted in error_string(): assert "test_exceptions.py(" in lines[3] assert lines[3].endswith("): __init__") - assert lines[4].endswith("): test_flaky_exception_failure_point_init") + assert lines[4].endswith( + "): _test_flaky_exception_failure_point_init_before_py_3_12" + ) + + +def _test_flaky_exception_failure_point_init_py_3_12(): + # Behavior change in Python 3.12: https://github.com/python/cpython/issues/102594 + what, py_err_set_after_what = m.error_already_set_what( + FlakyException, ("failure_point_init",) + ) + assert not py_err_set_after_what + lines = what.splitlines() + assert lines[0].endswith("ValueError[WITH __notes__]: triggered_failure_point_init") + assert lines[1] == "__notes__ (len=1):" + assert "Normalization failed:" in lines[2] + assert "FlakyException" in lines[2] + + +@pytest.mark.skipif( + "env.PYPY and sys.version_info[:2] < (3, 12)", + reason="PyErr_NormalizeException Segmentation fault", +) +def test_flaky_exception_failure_point_init(): + if sys.version_info[:2] < (3, 12): + _test_flaky_exception_failure_point_init_before_py_3_12() + else: + _test_flaky_exception_failure_point_init_py_3_12() def test_flaky_exception_failure_point_str(): @@ -327,10 +366,7 @@ def test_flaky_exception_failure_point_str(): ) assert not py_err_set_after_what lines = what.splitlines() - if env.PYPY and len(lines) == 3: - n = 3 # Traceback is missing. - else: - n = 5 + n = 3 if env.PYPY and len(lines) == 3 else 5 assert ( lines[:n] == [ @@ -360,3 +396,18 @@ def test_error_already_set_double_restore(): "Internal error: pybind11::detail::error_fetch_and_normalize::restore()" " called a second time. ORIGINAL ERROR: ValueError: Random error." ) + + +def test_pypy_oserror_normalization(): + # https://github.com/pybind/pybind11/issues/4075 + what = m.test_pypy_oserror_normalization() + assert "this_filename_must_not_exist" in what + + +def test_fn_cast_int_exception(): + with pytest.raises(RuntimeError) as excinfo: + m.test_fn_cast_int(lambda: None) + + assert str(excinfo.value).startswith( + "Unable to cast Python instance of type to C++ type" + ) diff --git a/pybind11/tests/test_factory_constructors.py b/pybind11/tests/test_factory_constructors.py index 120a587..04df802 100644 --- a/pybind11/tests/test_factory_constructors.py +++ b/pybind11/tests/test_factory_constructors.py @@ -96,7 +96,7 @@ def test_init_factory_signature(msg): 3. __init__(self: m.factory_constructors.TestFactory1, arg0: m.factory_constructors.tag.pointer_tag) -> None 4. __init__(self: m.factory_constructors.TestFactory1, arg0: handle, arg1: int, arg2: handle) -> None - """ # noqa: E501 line too long + """ ) diff --git a/pybind11/tests/test_gil_scoped.cpp b/pybind11/tests/test_gil_scoped.cpp index 97efdc1..f136086 100644 --- a/pybind11/tests/test_gil_scoped.cpp +++ b/pybind11/tests/test_gil_scoped.cpp @@ -11,6 +11,13 @@ #include "pybind11_tests.h" +#include +#include + +#define CROSS_MODULE(Function) \ + auto cm = py::module_::import("cross_module_gil_utils"); \ + auto target = reinterpret_cast(PyLong_AsVoidPtr(cm.attr(Function).ptr())); + class VirtClass { public: virtual ~VirtClass() = default; @@ -28,6 +35,16 @@ class PyVirtClass : public VirtClass { }; TEST_SUBMODULE(gil_scoped, m) { + m.attr("defined_THREAD_SANITIZER") = +#if defined(THREAD_SANITIZER) + true; +#else + false; +#endif + + m.def("intentional_deadlock", + []() { std::thread([]() { py::gil_scoped_acquire gil_acquired; }).join(); }); + py::class_(m, "VirtClass") .def(py::init<>()) .def("virtual_func", &VirtClass::virtual_func) @@ -37,11 +54,91 @@ TEST_SUBMODULE(gil_scoped, m) { m.def("test_callback_std_func", [](const std::function &func) { func(); }); m.def("test_callback_virtual_func", [](VirtClass &virt) { virt.virtual_func(); }); m.def("test_callback_pure_virtual_func", [](VirtClass &virt) { virt.pure_virtual_func(); }); - m.def("test_cross_module_gil", []() { - auto cm = py::module_::import("cross_module_gil_utils"); - auto gil_acquire = reinterpret_cast( - PyLong_AsVoidPtr(cm.attr("gil_acquire_funcaddr").ptr())); + m.def("test_cross_module_gil_released", []() { + CROSS_MODULE("gil_acquire_funcaddr") + py::gil_scoped_release gil_release; + target(); + }); + m.def("test_cross_module_gil_acquired", []() { + CROSS_MODULE("gil_acquire_funcaddr") + py::gil_scoped_acquire gil_acquire; + target(); + }); + m.def("test_cross_module_gil_inner_custom_released", []() { + CROSS_MODULE("gil_acquire_inner_custom_funcaddr") + py::gil_scoped_release gil_release; + target(); + }); + m.def("test_cross_module_gil_inner_custom_acquired", []() { + CROSS_MODULE("gil_acquire_inner_custom_funcaddr") + py::gil_scoped_acquire gil_acquire; + target(); + }); + m.def("test_cross_module_gil_inner_pybind11_released", []() { + CROSS_MODULE("gil_acquire_inner_pybind11_funcaddr") py::gil_scoped_release gil_release; - gil_acquire(); + target(); + }); + m.def("test_cross_module_gil_inner_pybind11_acquired", []() { + CROSS_MODULE("gil_acquire_inner_pybind11_funcaddr") + py::gil_scoped_acquire gil_acquire; + target(); + }); + m.def("test_cross_module_gil_nested_custom_released", []() { + CROSS_MODULE("gil_acquire_nested_custom_funcaddr") + py::gil_scoped_release gil_release; + target(); + }); + m.def("test_cross_module_gil_nested_custom_acquired", []() { + CROSS_MODULE("gil_acquire_nested_custom_funcaddr") + py::gil_scoped_acquire gil_acquire; + target(); + }); + m.def("test_cross_module_gil_nested_pybind11_released", []() { + CROSS_MODULE("gil_acquire_nested_pybind11_funcaddr") + py::gil_scoped_release gil_release; + target(); + }); + m.def("test_cross_module_gil_nested_pybind11_acquired", []() { + CROSS_MODULE("gil_acquire_nested_pybind11_funcaddr") + py::gil_scoped_acquire gil_acquire; + target(); + }); + m.def("test_release_acquire", [](const py::object &obj) { + py::gil_scoped_release gil_released; + py::gil_scoped_acquire gil_acquired; + return py::str(obj); + }); + m.def("test_nested_acquire", [](const py::object &obj) { + py::gil_scoped_release gil_released; + py::gil_scoped_acquire gil_acquired_outer; + py::gil_scoped_acquire gil_acquired_inner; + return py::str(obj); + }); + m.def("test_multi_acquire_release_cross_module", [](unsigned bits) { + py::set internals_ids; + internals_ids.add(PYBIND11_INTERNALS_ID); + { + py::gil_scoped_release gil_released; + auto thread_f = [bits, &internals_ids]() { + py::gil_scoped_acquire gil_acquired; + auto cm = py::module_::import("cross_module_gil_utils"); + auto target = reinterpret_cast( + PyLong_AsVoidPtr(cm.attr("gil_multi_acquire_release_funcaddr").ptr())); + std::string cm_internals_id = target(bits >> 3); + internals_ids.add(cm_internals_id); + }; + if ((bits & 0x1u) != 0u) { + thread_f(); + } + if ((bits & 0x2u) != 0u) { + std::thread non_python_thread(thread_f); + non_python_thread.join(); + } + if ((bits & 0x4u) != 0u) { + thread_f(); + } + } + return internals_ids; }); } diff --git a/pybind11/tests/test_gil_scoped.py b/pybind11/tests/test_gil_scoped.py index 52374b0..fc8af9b 100644 --- a/pybind11/tests/test_gil_scoped.py +++ b/pybind11/tests/test_gil_scoped.py @@ -1,45 +1,197 @@ import multiprocessing +import sys import threading +import time +import pytest + +import env from pybind11_tests import gil_scoped as m +class ExtendedVirtClass(m.VirtClass): + def virtual_func(self): + pass + + def pure_virtual_func(self): + pass + + +def test_callback_py_obj(): + m.test_callback_py_obj(lambda: None) + + +def test_callback_std_func(): + m.test_callback_std_func(lambda: None) + + +def test_callback_virtual_func(): + extended = ExtendedVirtClass() + m.test_callback_virtual_func(extended) + + +def test_callback_pure_virtual_func(): + extended = ExtendedVirtClass() + m.test_callback_pure_virtual_func(extended) + + +def test_cross_module_gil_released(): + """Makes sure that the GIL can be acquired by another module from a GIL-released state.""" + m.test_cross_module_gil_released() # Should not raise a SIGSEGV + + +def test_cross_module_gil_acquired(): + """Makes sure that the GIL can be acquired by another module from a GIL-acquired state.""" + m.test_cross_module_gil_acquired() # Should not raise a SIGSEGV + + +def test_cross_module_gil_inner_custom_released(): + """Makes sure that the GIL can be acquired/released by another module + from a GIL-released state using custom locking logic.""" + m.test_cross_module_gil_inner_custom_released() + + +def test_cross_module_gil_inner_custom_acquired(): + """Makes sure that the GIL can be acquired/acquired by another module + from a GIL-acquired state using custom locking logic.""" + m.test_cross_module_gil_inner_custom_acquired() + + +def test_cross_module_gil_inner_pybind11_released(): + """Makes sure that the GIL can be acquired/released by another module + from a GIL-released state using pybind11 locking logic.""" + m.test_cross_module_gil_inner_pybind11_released() + + +def test_cross_module_gil_inner_pybind11_acquired(): + """Makes sure that the GIL can be acquired/acquired by another module + from a GIL-acquired state using pybind11 locking logic.""" + m.test_cross_module_gil_inner_pybind11_acquired() + + +def test_cross_module_gil_nested_custom_released(): + """Makes sure that the GIL can be nested acquired/released by another module + from a GIL-released state using custom locking logic.""" + m.test_cross_module_gil_nested_custom_released() + + +def test_cross_module_gil_nested_custom_acquired(): + """Makes sure that the GIL can be nested acquired/acquired by another module + from a GIL-acquired state using custom locking logic.""" + m.test_cross_module_gil_nested_custom_acquired() + + +def test_cross_module_gil_nested_pybind11_released(): + """Makes sure that the GIL can be nested acquired/released by another module + from a GIL-released state using pybind11 locking logic.""" + m.test_cross_module_gil_nested_pybind11_released() + + +def test_cross_module_gil_nested_pybind11_acquired(): + """Makes sure that the GIL can be nested acquired/acquired by another module + from a GIL-acquired state using pybind11 locking logic.""" + m.test_cross_module_gil_nested_pybind11_acquired() + + +def test_release_acquire(): + assert m.test_release_acquire(0xAB) == "171" + + +def test_nested_acquire(): + assert m.test_nested_acquire(0xAB) == "171" + + +def test_multi_acquire_release_cross_module(): + for bits in range(16 * 8): + internals_ids = m.test_multi_acquire_release_cross_module(bits) + assert len(internals_ids) == 2 if bits % 8 else 1 + + +# Intentionally putting human review in the loop here, to guard against accidents. +VARS_BEFORE_ALL_BASIC_TESTS = dict(vars()) # Make a copy of the dict (critical). +ALL_BASIC_TESTS = ( + test_callback_py_obj, + test_callback_std_func, + test_callback_virtual_func, + test_callback_pure_virtual_func, + test_cross_module_gil_released, + test_cross_module_gil_acquired, + test_cross_module_gil_inner_custom_released, + test_cross_module_gil_inner_custom_acquired, + test_cross_module_gil_inner_pybind11_released, + test_cross_module_gil_inner_pybind11_acquired, + test_cross_module_gil_nested_custom_released, + test_cross_module_gil_nested_custom_acquired, + test_cross_module_gil_nested_pybind11_released, + test_cross_module_gil_nested_pybind11_acquired, + test_release_acquire, + test_nested_acquire, + test_multi_acquire_release_cross_module, +) + + +def test_all_basic_tests_completeness(): + num_found = 0 + for key, value in VARS_BEFORE_ALL_BASIC_TESTS.items(): + if not key.startswith("test_"): + continue + assert value in ALL_BASIC_TESTS + num_found += 1 + assert len(ALL_BASIC_TESTS) == num_found + + +def _intentional_deadlock(): + m.intentional_deadlock() + + +ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK = ALL_BASIC_TESTS + (_intentional_deadlock,) + + def _run_in_process(target, *args, **kwargs): - """Runs target in process and returns its exitcode after 10s (None if still alive).""" + test_fn = target if len(args) == 0 else args[0] + # Do not need to wait much, 10s should be more than enough. + timeout = 0.1 if test_fn is _intentional_deadlock else 10 process = multiprocessing.Process(target=target, args=args, kwargs=kwargs) process.daemon = True try: + t_start = time.time() process.start() - # Do not need to wait much, 10s should be more than enough. - process.join(timeout=10) + if timeout >= 100: # For debugging. + print( + "\nprocess.pid STARTED", process.pid, (sys.argv, target, args, kwargs) + ) + print(f"COPY-PASTE-THIS: gdb {sys.argv[0]} -p {process.pid}", flush=True) + process.join(timeout=timeout) + if timeout >= 100: + print("\nprocess.pid JOINED", process.pid, flush=True) + t_delta = time.time() - t_start + if process.exitcode == 66 and m.defined_THREAD_SANITIZER: # Issue #2754 + # WOULD-BE-NICE-TO-HAVE: Check that the message below is actually in the output. + # Maybe this could work: + # https://gist.github.com/alexeygrigorev/01ce847f2e721b513b42ea4a6c96905e + pytest.skip( + "ThreadSanitizer: starting new threads after multi-threaded fork is not supported." + ) + elif test_fn is _intentional_deadlock: + assert process.exitcode is None + return 0 + + if process.exitcode is None: + assert t_delta > 0.9 * timeout + msg = "DEADLOCK, most likely, exactly what this test is meant to detect." + if env.PYPY and env.WIN: + pytest.skip(msg) + raise RuntimeError(msg) return process.exitcode finally: if process.is_alive(): process.terminate() -def _python_to_cpp_to_python(): - """Calls different C++ functions that come back to Python.""" - - class ExtendedVirtClass(m.VirtClass): - def virtual_func(self): - pass - - def pure_virtual_func(self): - pass - - extended = ExtendedVirtClass() - m.test_callback_py_obj(lambda: None) - m.test_callback_std_func(lambda: None) - m.test_callback_virtual_func(extended) - m.test_callback_pure_virtual_func(extended) - - -def _python_to_cpp_to_python_from_threads(num_threads, parallel=False): - """Calls different C++ functions that come back to Python, from Python threads.""" +def _run_in_threads(test_fn, num_threads, parallel): threads = [] for _ in range(num_threads): - thread = threading.Thread(target=_python_to_cpp_to_python) + thread = threading.Thread(target=test_fn) thread.daemon = True thread.start() if parallel: @@ -51,43 +203,40 @@ def _python_to_cpp_to_python_from_threads(num_threads, parallel=False): # TODO: FIXME, sometimes returns -11 (segfault) instead of 0 on macOS Python 3.9 -def test_python_to_cpp_to_python_from_thread(): +@pytest.mark.parametrize("test_fn", ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK) +def test_run_in_process_one_thread(test_fn): """Makes sure there is no GIL deadlock when running in a thread. It runs in a separate process to be able to stop and assert if it deadlocks. """ - assert _run_in_process(_python_to_cpp_to_python_from_threads, 1) == 0 + assert _run_in_process(_run_in_threads, test_fn, num_threads=1, parallel=False) == 0 # TODO: FIXME on macOS Python 3.9 -def test_python_to_cpp_to_python_from_thread_multiple_parallel(): +@pytest.mark.parametrize("test_fn", ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK) +def test_run_in_process_multiple_threads_parallel(test_fn): """Makes sure there is no GIL deadlock when running in a thread multiple times in parallel. It runs in a separate process to be able to stop and assert if it deadlocks. """ - assert _run_in_process(_python_to_cpp_to_python_from_threads, 8, parallel=True) == 0 + assert _run_in_process(_run_in_threads, test_fn, num_threads=8, parallel=True) == 0 # TODO: FIXME on macOS Python 3.9 -def test_python_to_cpp_to_python_from_thread_multiple_sequential(): +@pytest.mark.parametrize("test_fn", ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK) +def test_run_in_process_multiple_threads_sequential(test_fn): """Makes sure there is no GIL deadlock when running in a thread multiple times sequentially. It runs in a separate process to be able to stop and assert if it deadlocks. """ - assert ( - _run_in_process(_python_to_cpp_to_python_from_threads, 8, parallel=False) == 0 - ) + assert _run_in_process(_run_in_threads, test_fn, num_threads=8, parallel=False) == 0 # TODO: FIXME on macOS Python 3.9 -def test_python_to_cpp_to_python_from_process(): +@pytest.mark.parametrize("test_fn", ALL_BASIC_TESTS_PLUS_INTENTIONAL_DEADLOCK) +def test_run_in_process_direct(test_fn): """Makes sure there is no GIL deadlock when using processes. This test is for completion, but it was never an issue. """ - assert _run_in_process(_python_to_cpp_to_python) == 0 - - -def test_cross_module_gil(): - """Makes sure that the GIL can be acquired by another module from a GIL-released state.""" - m.test_cross_module_gil() # Should not raise a SIGSEGV + assert _run_in_process(test_fn) == 0 diff --git a/pybind11/tests/test_iostream.py b/pybind11/tests/test_iostream.py index 5bbdf69..d283eb1 100644 --- a/pybind11/tests/test_iostream.py +++ b/pybind11/tests/test_iostream.py @@ -9,16 +9,16 @@ def test_captured(capsys): m.captured_output(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr m.captured_err(msg) stdout, stderr = capsys.readouterr() - assert stdout == "" + assert not stdout assert stderr == msg @@ -30,7 +30,7 @@ def test_captured_large_string(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_captured_utf8_2byte_offset0(capsys): @@ -40,7 +40,7 @@ def test_captured_utf8_2byte_offset0(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_captured_utf8_2byte_offset1(capsys): @@ -50,7 +50,7 @@ def test_captured_utf8_2byte_offset1(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_captured_utf8_3byte_offset0(capsys): @@ -60,7 +60,7 @@ def test_captured_utf8_3byte_offset0(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_captured_utf8_3byte_offset1(capsys): @@ -70,7 +70,7 @@ def test_captured_utf8_3byte_offset1(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_captured_utf8_3byte_offset2(capsys): @@ -80,7 +80,7 @@ def test_captured_utf8_3byte_offset2(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_captured_utf8_4byte_offset0(capsys): @@ -90,7 +90,7 @@ def test_captured_utf8_4byte_offset0(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_captured_utf8_4byte_offset1(capsys): @@ -100,7 +100,7 @@ def test_captured_utf8_4byte_offset1(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_captured_utf8_4byte_offset2(capsys): @@ -110,7 +110,7 @@ def test_captured_utf8_4byte_offset2(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_captured_utf8_4byte_offset3(capsys): @@ -120,7 +120,7 @@ def test_captured_utf8_4byte_offset3(capsys): m.captured_output_default(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_guard_capture(capsys): @@ -128,7 +128,7 @@ def test_guard_capture(capsys): m.guard_output(msg) stdout, stderr = capsys.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr def test_series_captured(capture): @@ -145,7 +145,7 @@ def test_flush(capfd): with m.ostream_redirect(): m.noisy_function(msg, flush=False) stdout, stderr = capfd.readouterr() - assert stdout == "" + assert not stdout m.noisy_function(msg2, flush=True) stdout, stderr = capfd.readouterr() @@ -164,15 +164,15 @@ def test_not_captured(capfd): m.raw_output(msg) stdout, stderr = capfd.readouterr() assert stdout == msg - assert stderr == "" - assert stream.getvalue() == "" + assert not stderr + assert not stream.getvalue() stream = StringIO() with redirect_stdout(stream): m.captured_output(msg) stdout, stderr = capfd.readouterr() - assert stdout == "" - assert stderr == "" + assert not stdout + assert not stderr assert stream.getvalue() == msg @@ -182,16 +182,16 @@ def test_err(capfd): with redirect_stderr(stream): m.raw_err(msg) stdout, stderr = capfd.readouterr() - assert stdout == "" + assert not stdout assert stderr == msg - assert stream.getvalue() == "" + assert not stream.getvalue() stream = StringIO() with redirect_stderr(stream): m.captured_err(msg) stdout, stderr = capfd.readouterr() - assert stdout == "" - assert stderr == "" + assert not stdout + assert not stderr assert stream.getvalue() == msg @@ -221,14 +221,13 @@ def test_redirect(capfd): m.raw_output(msg) stdout, stderr = capfd.readouterr() assert stdout == msg - assert stream.getvalue() == "" + assert not stream.getvalue() stream = StringIO() - with redirect_stdout(stream): - with m.ostream_redirect(): - m.raw_output(msg) + with redirect_stdout(stream), m.ostream_redirect(): + m.raw_output(msg) stdout, stderr = capfd.readouterr() - assert stdout == "" + assert not stdout assert stream.getvalue() == msg stream = StringIO() @@ -236,7 +235,7 @@ def test_redirect(capfd): m.raw_output(msg) stdout, stderr = capfd.readouterr() assert stdout == msg - assert stream.getvalue() == "" + assert not stream.getvalue() def test_redirect_err(capfd): @@ -244,13 +243,12 @@ def test_redirect_err(capfd): msg2 = "StdErr" stream = StringIO() - with redirect_stderr(stream): - with m.ostream_redirect(stdout=False): - m.raw_output(msg) - m.raw_err(msg2) + with redirect_stderr(stream), m.ostream_redirect(stdout=False): + m.raw_output(msg) + m.raw_err(msg2) stdout, stderr = capfd.readouterr() assert stdout == msg - assert stderr == "" + assert not stderr assert stream.getvalue() == msg2 @@ -260,14 +258,12 @@ def test_redirect_both(capfd): stream = StringIO() stream2 = StringIO() - with redirect_stdout(stream): - with redirect_stderr(stream2): - with m.ostream_redirect(): - m.raw_output(msg) - m.raw_err(msg2) + with redirect_stdout(stream), redirect_stderr(stream2), m.ostream_redirect(): + m.raw_output(msg) + m.raw_err(msg2) stdout, stderr = capfd.readouterr() - assert stdout == "" - assert stderr == "" + assert not stdout + assert not stderr assert stream.getvalue() == msg assert stream2.getvalue() == msg2 diff --git a/pybind11/tests/test_kwargs_and_defaults.cpp b/pybind11/tests/test_kwargs_and_defaults.cpp index 7418afe..77e72c0 100644 --- a/pybind11/tests/test_kwargs_and_defaults.cpp +++ b/pybind11/tests/test_kwargs_and_defaults.cpp @@ -43,7 +43,15 @@ TEST_SUBMODULE(kwargs_and_defaults, m) { m.def("kw_func_udl_z", kw_func, "x"_a, "y"_a = 0); // test_args_and_kwargs - m.def("args_function", [](py::args args) -> py::tuple { return std::move(args); }); + m.def("args_function", [](py::args args) -> py::tuple { + PYBIND11_WARNING_PUSH + +#ifdef PYBIND11_DETECTED_CLANG_WITH_MISLEADING_CALL_STD_MOVE_EXPLICITLY_WARNING + PYBIND11_WARNING_DISABLE_CLANG("-Wreturn-std-move") +#endif + return args; + PYBIND11_WARNING_POP + }); m.def("args_kwargs_function", [](const py::args &args, const py::kwargs &kwargs) { return py::make_tuple(args, kwargs); }); diff --git a/pybind11/tests/test_kwargs_and_defaults.py b/pybind11/tests/test_kwargs_and_defaults.py index ab70178..7174726 100644 --- a/pybind11/tests/test_kwargs_and_defaults.py +++ b/pybind11/tests/test_kwargs_and_defaults.py @@ -25,7 +25,7 @@ def test_function_signatures(doc): ) -def test_named_arguments(msg): +def test_named_arguments(): assert m.kw_func0(5, 10) == "x=5, y=10" assert m.kw_func1(5, 10) == "x=5, y=10" @@ -43,8 +43,7 @@ def test_named_arguments(msg): # noinspection PyArgumentList m.kw_func2(x=5, y=10, z=12) assert excinfo.match( - r"(?s)^kw_func2\(\): incompatible.*Invoked with: kwargs: ((x=5|y=10|z=12)(, |$))" - + "{3}$" + r"(?s)^kw_func2\(\): incompatible.*Invoked with: kwargs: ((x=5|y=10|z=12)(, |$)){3}$" ) assert m.kw_func4() == "{13 17}" @@ -59,7 +58,7 @@ def test_arg_and_kwargs(): assert m.args_function(*args) == args args = "a1", "a2" - kwargs = dict(arg3="a3", arg4=4) + kwargs = {"arg3": "a3", "arg4": 4} assert m.args_kwargs_function(*args, **kwargs) == (args, kwargs) @@ -177,7 +176,7 @@ def test_mixed_args_and_kwargs(msg): assert ( m.args_kwonly_kwargs_defaults.__doc__ - == "args_kwonly_kwargs_defaults(i: int = 1, j: float = 3.14159, *args, z: int = 42, **kwargs) -> tuple\n" # noqa: E501 line too long + == "args_kwonly_kwargs_defaults(i: int = 1, j: float = 3.14159, *args, z: int = 42, **kwargs) -> tuple\n" ) assert m.args_kwonly_kwargs_defaults() == (1, 3.14159, (), 42, {}) assert m.args_kwonly_kwargs_defaults(2) == (2, 3.14159, (), 42, {}) @@ -233,15 +232,15 @@ def test_keyword_only_args(msg): x.method(i=1, j=2) assert ( m.first_arg_kw_only.__init__.__doc__ - == "__init__(self: pybind11_tests.kwargs_and_defaults.first_arg_kw_only, *, i: int = 0) -> None\n" # noqa: E501 line too long + == "__init__(self: pybind11_tests.kwargs_and_defaults.first_arg_kw_only, *, i: int = 0) -> None\n" ) assert ( m.first_arg_kw_only.method.__doc__ - == "method(self: pybind11_tests.kwargs_and_defaults.first_arg_kw_only, *, i: int = 1, j: int = 2) -> None\n" # noqa: E501 line too long + == "method(self: pybind11_tests.kwargs_and_defaults.first_arg_kw_only, *, i: int = 1, j: int = 2) -> None\n" ) -def test_positional_only_args(msg): +def test_positional_only_args(): assert m.pos_only_all(1, 2) == (1, 2) assert m.pos_only_all(2, 1) == (2, 1) @@ -283,7 +282,7 @@ def test_positional_only_args(msg): # Mix it with args and kwargs: assert ( m.args_kwonly_full_monty.__doc__ - == "args_kwonly_full_monty(arg0: int = 1, arg1: int = 2, /, j: float = 3.14159, *args, z: int = 42, **kwargs) -> tuple\n" # noqa: E501 line too long + == "args_kwonly_full_monty(arg0: int = 1, arg1: int = 2, /, j: float = 3.14159, *args, z: int = 42, **kwargs) -> tuple\n" ) assert m.args_kwonly_full_monty() == (1, 2, 3.14159, (), 42, {}) assert m.args_kwonly_full_monty(8) == (8, 2, 3.14159, (), 42, {}) @@ -326,18 +325,18 @@ def test_positional_only_args(msg): # https://github.com/pybind/pybind11/pull/3402#issuecomment-963341987 assert ( m.first_arg_kw_only.pos_only.__doc__ - == "pos_only(self: pybind11_tests.kwargs_and_defaults.first_arg_kw_only, /, i: int, j: int) -> None\n" # noqa: E501 line too long + == "pos_only(self: pybind11_tests.kwargs_and_defaults.first_arg_kw_only, /, i: int, j: int) -> None\n" ) def test_signatures(): - assert "kw_only_all(*, i: int, j: int) -> tuple\n" == m.kw_only_all.__doc__ - assert "kw_only_mixed(i: int, *, j: int) -> tuple\n" == m.kw_only_mixed.__doc__ - assert "pos_only_all(i: int, j: int, /) -> tuple\n" == m.pos_only_all.__doc__ - assert "pos_only_mix(i: int, /, j: int) -> tuple\n" == m.pos_only_mix.__doc__ + assert m.kw_only_all.__doc__ == "kw_only_all(*, i: int, j: int) -> tuple\n" + assert m.kw_only_mixed.__doc__ == "kw_only_mixed(i: int, *, j: int) -> tuple\n" + assert m.pos_only_all.__doc__ == "pos_only_all(i: int, j: int, /) -> tuple\n" + assert m.pos_only_mix.__doc__ == "pos_only_mix(i: int, /, j: int) -> tuple\n" assert ( - "pos_kw_only_mix(i: int, /, j: int, *, k: int) -> tuple\n" - == m.pos_kw_only_mix.__doc__ + m.pos_kw_only_mix.__doc__ + == "pos_kw_only_mix(i: int, /, j: int, *, k: int) -> tuple\n" ) diff --git a/pybind11/tests/test_local_bindings.py b/pybind11/tests/test_local_bindings.py index 654d96d..d641877 100644 --- a/pybind11/tests/test_local_bindings.py +++ b/pybind11/tests/test_local_bindings.py @@ -130,7 +130,8 @@ def test_stl_bind_global(): def test_mixed_local_global(): """Local types take precedence over globally registered types: a module with a `module_local` type can be registered even if the type is already registered globally. With the module, - casting will go to the local type; outside the module casting goes to the global type.""" + casting will go to the local type; outside the module casting goes to the global type. + """ import pybind11_cross_module_tests as cm m.register_mixed_global() diff --git a/pybind11/tests/test_methods_and_attributes.cpp b/pybind11/tests/test_methods_and_attributes.cpp index 815dd5e..31d46eb 100644 --- a/pybind11/tests/test_methods_and_attributes.cpp +++ b/pybind11/tests/test_methods_and_attributes.cpp @@ -177,6 +177,38 @@ struct RValueRefParam { std::size_t func4(std::string &&s) const & { return s.size(); } }; +namespace pybind11_tests { +namespace exercise_is_setter { + +struct FieldBase { + int int_value() const { return int_value_; } + + FieldBase &SetIntValue(int int_value) { + int_value_ = int_value; + return *this; + } + +private: + int int_value_ = -99; +}; + +struct Field : FieldBase {}; + +void add_bindings(py::module &m) { + py::module sm = m.def_submodule("exercise_is_setter"); + // NOTE: FieldBase is not wrapped, therefore ... + py::class_(sm, "Field") + .def(py::init<>()) + .def_property( + "int_value", + &Field::int_value, + &Field::SetIntValue // ... the `FieldBase &` return value here cannot be converted. + ); +} + +} // namespace exercise_is_setter +} // namespace pybind11_tests + TEST_SUBMODULE(methods_and_attributes, m) { // test_methods_and_attributes py::class_ emna(m, "ExampleMandA"); @@ -456,4 +488,6 @@ TEST_SUBMODULE(methods_and_attributes, m) { .def("func2", &RValueRefParam::func2) .def("func3", &RValueRefParam::func3) .def("func4", &RValueRefParam::func4); + + pybind11_tests::exercise_is_setter::add_bindings(m); } diff --git a/pybind11/tests/test_methods_and_attributes.py b/pybind11/tests/test_methods_and_attributes.py index 0a2ae12..955a85f 100644 --- a/pybind11/tests/test_methods_and_attributes.py +++ b/pybind11/tests/test_methods_and_attributes.py @@ -183,9 +183,9 @@ def test_static_properties(): # Only static attributes can be deleted del m.TestPropertiesOverride.def_readonly_static + assert hasattr(m.TestPropertiesOverride, "def_readonly_static") assert ( - hasattr(m.TestPropertiesOverride, "def_readonly_static") - and m.TestPropertiesOverride.def_readonly_static + m.TestPropertiesOverride.def_readonly_static is m.TestProperties.def_readonly_static ) assert "def_readonly_static" not in m.TestPropertiesOverride.__dict__ @@ -256,10 +256,7 @@ def test_no_mixed_overloads(): @pytest.mark.parametrize("access", ["ro", "rw", "static_ro", "static_rw"]) def test_property_return_value_policies(access): - if not access.startswith("static"): - obj = m.TestPropRVP() - else: - obj = m.TestPropRVP + obj = m.TestPropRVP() if not access.startswith("static") else m.TestPropRVP ref = getattr(obj, access + "_ref") assert ref.value == 1 @@ -525,3 +522,12 @@ def test_rvalue_ref_param(): assert r.func2("1234") == 4 assert r.func3("12345") == 5 assert r.func4("123456") == 6 + + +def test_is_setter(): + fld = m.exercise_is_setter.Field() + assert fld.int_value == -99 + setter_return = fld.int_value = 100 + assert isinstance(setter_return, int) + assert setter_return == 100 + assert fld.int_value == 100 diff --git a/pybind11/tests/test_modules.py b/pybind11/tests/test_modules.py index e11d68e..2f6d825 100644 --- a/pybind11/tests/test_modules.py +++ b/pybind11/tests/test_modules.py @@ -1,3 +1,5 @@ +import builtins + import pytest import env @@ -61,7 +63,6 @@ def test_importing(): from pybind11_tests.modules import OD assert OD is OrderedDict - assert str(OD([(1, "a"), (2, "b")])) == "OrderedDict([(1, 'a'), (2, 'b')])" def test_pydoc(): @@ -86,12 +87,7 @@ def test_builtin_key_type(): Previous versions of pybind11 would add a unicode key in python 2. """ - if hasattr(__builtins__, "keys"): - keys = __builtins__.keys() - else: # this is to make pypy happy since builtins is different there. - keys = __builtins__.__dict__.keys() - - assert {type(k) for k in keys} == {str} + assert all(type(k) == str for k in dir(builtins)) @pytest.mark.xfail("env.PYPY", reason="PyModule_GetName()") @@ -107,11 +103,10 @@ def test_def_submodule_failures(): sm_name_orig = sm.__name__ sm.__name__ = malformed_utf8 try: - with pytest.raises(Exception): - # Seen with Python 3.9: SystemError: nameless module - # But we do not want to exercise the internals of PyModule_GetName(), which could - # change in future versions of Python, but a bad __name__ is very likely to cause - # some kind of failure indefinitely. + # We want to assert that a bad __name__ causes some kind of failure, although we do not want to exercise + # the internals of PyModule_GetName(). Currently all supported Python versions raise SystemError. If that + # changes in future Python versions, simply add the new expected exception types here. + with pytest.raises(SystemError): m.def_submodule(sm, b"SubSubModuleName") finally: # Clean up to ensure nothing gets upset by a module with an invalid __name__. diff --git a/pybind11/tests/test_numpy_array.cpp b/pybind11/tests/test_numpy_array.cpp index 69ddbe1..8c122a8 100644 --- a/pybind11/tests/test_numpy_array.cpp +++ b/pybind11/tests/test_numpy_array.cpp @@ -521,4 +521,32 @@ TEST_SUBMODULE(numpy_array, sm) { sm.def("test_fmt_desc_double", [](const py::array_t &) {}); sm.def("test_fmt_desc_const_float", [](const py::array_t &) {}); sm.def("test_fmt_desc_const_double", [](const py::array_t &) {}); + + sm.def("round_trip_float", [](double d) { return d; }); + + sm.def("pass_array_pyobject_ptr_return_sum_str_values", + [](const py::array_t &objs) { + std::string sum_str_values; + for (const auto &obj : objs) { + sum_str_values += py::str(obj.attr("value")); + } + return sum_str_values; + }); + + sm.def("pass_array_pyobject_ptr_return_as_list", + [](const py::array_t &objs) -> py::list { return objs; }); + + sm.def("return_array_pyobject_ptr_cpp_loop", [](const py::list &objs) { + py::size_t arr_size = py::len(objs); + py::array_t arr_from_list(static_cast(arr_size)); + PyObject **data = arr_from_list.mutable_data(); + for (py::size_t i = 0; i < arr_size; i++) { + assert(data[i] == nullptr); + data[i] = py::cast(objs[i].attr("value")); + } + return arr_from_list; + }); + + sm.def("return_array_pyobject_ptr_from_list", + [](const py::list &objs) -> py::array_t { return objs; }); } diff --git a/pybind11/tests/test_numpy_array.py b/pybind11/tests/test_numpy_array.py index 504963b..12e7d17 100644 --- a/pybind11/tests/test_numpy_array.py +++ b/pybind11/tests/test_numpy_array.py @@ -22,7 +22,7 @@ def test_dtypes(): ) -@pytest.fixture(scope="function") +@pytest.fixture() def arr(): return np.array([[1, 2, 3], [4, 5, 6]], "=u2") @@ -67,7 +67,7 @@ def test_array_attributes(): @pytest.mark.parametrize( - "args, ret", [([], 0), ([0], 0), ([1], 3), ([0, 1], 1), ([1, 2], 5)] + ("args", "ret"), [([], 0), ([0], 0), ([1], 3), ([0, 1], 1), ([1, 2], 5)] ) def test_index_offset(arr, args, ret): assert m.index_at(arr, *args) == ret @@ -93,7 +93,7 @@ def test_dim_check_fail(arr): @pytest.mark.parametrize( - "args, ret", + ("args", "ret"), [ ([], [1, 2, 3, 4, 5, 6]), ([1], [4, 5, 6]), @@ -211,12 +211,14 @@ def assert_references(a, b, base=None): assert b[0, 0] == 1234 a1 = np.array([1, 2], dtype=np.int16) - assert a1.flags.owndata and a1.base is None + assert a1.flags.owndata + assert a1.base is None a2 = m.wrap(a1) assert_references(a1, a2) a1 = np.array([[1, 2], [3, 4]], dtype=np.float32, order="F") - assert a1.flags.owndata and a1.base is None + assert a1.flags.owndata + assert a1.base is None a2 = m.wrap(a1) assert_references(a1, a2) @@ -451,13 +453,15 @@ def test_array_resize(): try: m.array_resize3(a, 3, True) except ValueError as e: - assert str(e).startswith("cannot resize an array") + assert str(e).startswith("cannot resize an array") # noqa: PT017 # transposed array doesn't own data b = a.transpose() try: m.array_resize3(b, 3, False) except ValueError as e: - assert str(e).startswith("cannot resize this array: it does not own its data") + assert str(e).startswith( # noqa: PT017 + "cannot resize this array: it does not own its data" + ) # ... but reshape should be fine m.array_reshape2(b) assert b.shape == (8, 8) @@ -585,3 +589,80 @@ def test_dtype_refcount_leak(): m.ndim(a) after = getrefcount(dtype) assert after == before + + +def test_round_trip_float(): + arr = np.zeros((), np.float64) + arr[()] = 37.2 + assert m.round_trip_float(arr) == 37.2 + + +# HINT: An easy and robust way (although only manual unfortunately) to check for +# ref-count leaks in the test_.*pyobject_ptr.* functions below is to +# * temporarily insert `while True:` (one-by-one), +# * run this test, and +# * run the Linux `top` command in another shell to visually monitor +# `RES` for a minute or two. +# If there is a leak, it is usually evident in seconds because the `RES` +# value increases without bounds. (Don't forget to Ctrl-C the test!) + + +# For use as a temporary user-defined object, to maximize sensitivity of the tests below: +# * Ref-count leaks will be immediately evident. +# * Sanitizers are much more likely to detect heap-use-after-free due to +# other ref-count bugs. +class PyValueHolder: + def __init__(self, value): + self.value = value + + +def WrapWithPyValueHolder(*values): + return [PyValueHolder(v) for v in values] + + +def UnwrapPyValueHolder(vhs): + return [vh.value for vh in vhs] + + +def test_pass_array_pyobject_ptr_return_sum_str_values_ndarray(): + # Intentionally all temporaries, do not change. + assert ( + m.pass_array_pyobject_ptr_return_sum_str_values( + np.array(WrapWithPyValueHolder(-3, "four", 5.0), dtype=object) + ) + == "-3four5.0" + ) + + +def test_pass_array_pyobject_ptr_return_sum_str_values_list(): + # Intentionally all temporaries, do not change. + assert ( + m.pass_array_pyobject_ptr_return_sum_str_values( + WrapWithPyValueHolder(2, "three", -4.0) + ) + == "2three-4.0" + ) + + +def test_pass_array_pyobject_ptr_return_as_list(): + # Intentionally all temporaries, do not change. + assert UnwrapPyValueHolder( + m.pass_array_pyobject_ptr_return_as_list( + np.array(WrapWithPyValueHolder(-1, "two", 3.0), dtype=object) + ) + ) == [-1, "two", 3.0] + + +@pytest.mark.parametrize( + ("return_array_pyobject_ptr", "unwrap"), + [ + (m.return_array_pyobject_ptr_cpp_loop, list), + (m.return_array_pyobject_ptr_from_list, UnwrapPyValueHolder), + ], +) +def test_return_array_pyobject_ptr_cpp_loop(return_array_pyobject_ptr, unwrap): + # Intentionally all temporaries, do not change. + arr_from_list = return_array_pyobject_ptr(WrapWithPyValueHolder(6, "seven", -8.0)) + assert isinstance(arr_from_list, np.ndarray) + assert arr_from_list.dtype == np.dtype("O") + assert unwrap(arr_from_list) == [6, "seven", -8.0] diff --git a/pybind11/tests/test_numpy_dtypes.py b/pybind11/tests/test_numpy_dtypes.py index fcfd587..d10457e 100644 --- a/pybind11/tests/test_numpy_dtypes.py +++ b/pybind11/tests/test_numpy_dtypes.py @@ -130,14 +130,10 @@ def test_dtype(simple_dtype): partial_nested_fmt(), "[('a','S3'),('b','S3')]", ( - "{{'names':['a','b','c','d']," - + "'formats':[('S4',(3,)),('" - + e - + "i4',(2,)),('u1',(3,)),('" - + e - + "f4',(4,2))]," - + "'offsets':[0,12,20,24],'itemsize':56}}" - ).format(e=e), + "{'names':['a','b','c','d']," + f"'formats':[('S4',(3,)),('{e}i4',(2,)),('u1',(3,)),('{e}f4',(4,2))]," + "'offsets':[0,12,20,24],'itemsize':56}" + ), "[('e1','" + e + "i8'),('e2','u1')]", "[('x','i1'),('y','" + e + "u8')]", "[('cflt','" + e + "c8'),('cdbl','" + e + "c16')]", @@ -291,19 +287,17 @@ def test_array_array(): arr = m.create_array_array(3) assert str(arr.dtype).replace(" ", "") == ( - "{{'names':['a','b','c','d']," - + "'formats':[('S4',(3,)),('" - + e - + "i4',(2,)),('u1',(3,)),('{e}f4',(4,2))]," - + "'offsets':[0,12,20,24],'itemsize':56}}" - ).format(e=e) + "{'names':['a','b','c','d']," + f"'formats':[('S4',(3,)),('{e}i4',(2,)),('u1',(3,)),('{e}f4',(4,2))]," + "'offsets':[0,12,20,24],'itemsize':56}" + ) assert m.print_array_array(arr) == [ "a={{A,B,C,D},{K,L,M,N},{U,V,W,X}},b={0,1}," - + "c={0,1,2},d={{0,1},{10,11},{20,21},{30,31}}", + "c={0,1,2},d={{0,1},{10,11},{20,21},{30,31}}", "a={{W,X,Y,Z},{G,H,I,J},{Q,R,S,T}},b={1000,1001}," - + "c={10,11,12},d={{100,101},{110,111},{120,121},{130,131}}", + "c={10,11,12},d={{100,101},{110,111},{120,121},{130,131}}", "a={{S,T,U,V},{C,D,E,F},{M,N,O,P}},b={2000,2001}," - + "c={20,21,22},d={{200,201},{210,211},{220,221},{230,231}}", + "c={20,21,22},d={{200,201},{210,211},{220,221},{230,231}}", ] assert arr["a"].tolist() == [ [b"ABCD", b"KLMN", b"UVWX"], diff --git a/pybind11/tests/test_numpy_vectorize.py b/pybind11/tests/test_numpy_vectorize.py index 7e8c015..f1e8b62 100644 --- a/pybind11/tests/test_numpy_vectorize.py +++ b/pybind11/tests/test_numpy_vectorize.py @@ -149,7 +149,7 @@ def test_docs(doc): doc(m.vectorized_func) == """ vectorized_func(arg0: numpy.ndarray[numpy.int32], arg1: numpy.ndarray[numpy.float32], arg2: numpy.ndarray[numpy.float64]) -> object - """ # noqa: E501 line too long + """ ) diff --git a/pybind11/tests/test_operator_overloading.cpp b/pybind11/tests/test_operator_overloading.cpp index a4b895a..112a363 100644 --- a/pybind11/tests/test_operator_overloading.cpp +++ b/pybind11/tests/test_operator_overloading.cpp @@ -132,22 +132,18 @@ struct hash { // Not a good abs function, but easy to test. std::string abs(const Vector2 &) { return "abs(Vector2)"; } -// MSVC & Intel warns about unknown pragmas, and warnings are errors. -#if !defined(_MSC_VER) && !defined(__INTEL_COMPILER) -# pragma GCC diagnostic push // clang 7.0.0 and Apple LLVM 10.0.1 introduce `-Wself-assign-overloaded` to // `-Wall`, which is used here for overloading (e.g. `py::self += py::self `). -// Here, we suppress the warning using `#pragma diagnostic`. +// Here, we suppress the warning // Taken from: https://github.com/RobotLocomotion/drake/commit/aaf84b46 // TODO(eric): This could be resolved using a function / functor (e.g. `py::self()`). -# if defined(__APPLE__) && defined(__clang__) -# if (__clang_major__ >= 10) -# pragma GCC diagnostic ignored "-Wself-assign-overloaded" -# endif -# elif defined(__clang__) -# if (__clang_major__ >= 7) -# pragma GCC diagnostic ignored "-Wself-assign-overloaded" -# endif +#if defined(__APPLE__) && defined(__clang__) +# if (__clang_major__ >= 10) +PYBIND11_WARNING_DISABLE_CLANG("-Wself-assign-overloaded") +# endif +#elif defined(__clang__) +# if (__clang_major__ >= 7) +PYBIND11_WARNING_DISABLE_CLANG("-Wself-assign-overloaded") # endif #endif @@ -283,6 +279,3 @@ TEST_SUBMODULE(operators, m) { m.def("get_unhashable_HashMe_set", []() { return std::unordered_set{{"one"}}; }); } -#if !defined(_MSC_VER) && !defined(__INTEL_COMPILER) -# pragma GCC diagnostic pop -#endif diff --git a/pybind11/tests/test_operator_overloading.py b/pybind11/tests/test_operator_overloading.py index b228da3..9fde305 100644 --- a/pybind11/tests/test_operator_overloading.py +++ b/pybind11/tests/test_operator_overloading.py @@ -130,7 +130,6 @@ def test_nested(): def test_overriding_eq_reset_hash(): - assert m.Comparable(15) is not m.Comparable(15) assert m.Comparable(15) == m.Comparable(15) diff --git a/pybind11/tests/test_pytypes.cpp b/pybind11/tests/test_pytypes.cpp index f532e26..b4ee642 100644 --- a/pybind11/tests/test_pytypes.cpp +++ b/pybind11/tests/test_pytypes.cpp @@ -99,6 +99,8 @@ void m_defs(py::module_ &m) { } // namespace handle_from_move_only_type_with_operator_PyObject TEST_SUBMODULE(pytypes, m) { + m.def("obj_class_name", [](py::handle obj) { return py::detail::obj_class_name(obj.ptr()); }); + handle_from_move_only_type_with_operator_PyObject::m_defs(m); // test_bool @@ -109,6 +111,11 @@ TEST_SUBMODULE(pytypes, m) { m.def("get_iterator", [] { return py::iterator(); }); // test_iterable m.def("get_iterable", [] { return py::iterable(); }); + m.def("get_frozenset_from_iterable", + [](const py::iterable &iter) { return py::frozenset(iter); }); + m.def("get_list_from_iterable", [](const py::iterable &iter) { return py::list(iter); }); + m.def("get_set_from_iterable", [](const py::iterable &iter) { return py::set(iter); }); + m.def("get_tuple_from_iterable", [](const py::iterable &iter) { return py::tuple(iter); }); // test_float m.def("get_float", [] { return py::float_(0.0f); }); // test_list @@ -178,7 +185,7 @@ TEST_SUBMODULE(pytypes, m) { return d2; }); m.def("dict_contains", - [](const py::dict &dict, py::object val) { return dict.contains(val); }); + [](const py::dict &dict, const py::object &val) { return dict.contains(val); }); m.def("dict_contains", [](const py::dict &dict, const char *val) { return dict.contains(val); }); @@ -201,7 +208,12 @@ TEST_SUBMODULE(pytypes, m) { m.def("str_from_char_ssize_t", []() { return py::str{"red", (py::ssize_t) 3}; }); m.def("str_from_char_size_t", []() { return py::str{"blue", (py::size_t) 4}; }); m.def("str_from_string", []() { return py::str(std::string("baz")); }); + m.def("str_from_std_string_input", [](const std::string &stri) { return py::str(stri); }); + m.def("str_from_cstr_input", [](const char *c_str) { return py::str(c_str); }); m.def("str_from_bytes", []() { return py::str(py::bytes("boo", 3)); }); + m.def("str_from_bytes_input", + [](const py::bytes &encoded_str) { return py::str(encoded_str); }); + m.def("str_from_object", [](const py::object &obj) { return py::str(obj); }); m.def("repr_from_object", [](const py::object &obj) { return py::repr(obj); }); m.def("str_from_handle", [](py::handle h) { return py::str(h); }); @@ -248,6 +260,15 @@ TEST_SUBMODULE(pytypes, m) { }); }); + m.def("return_capsule_with_destructor_3", []() { + py::print("creating capsule"); + auto cap = py::capsule((void *) 1233, "oname", [](void *ptr) { + py::print("destructing capsule: {}"_s.format((size_t) ptr)); + }); + py::print("original name: {}"_s.format(cap.name())); + return cap; + }); + m.def("return_renamed_capsule_with_destructor_2", []() { py::print("creating capsule"); auto cap = py::capsule((void *) 1234, [](void *ptr) { @@ -284,6 +305,12 @@ TEST_SUBMODULE(pytypes, m) { return capsule; }); + m.def("return_capsule_with_explicit_nullptr_dtor", []() { + py::print("creating capsule with explicit nullptr dtor"); + return py::capsule(reinterpret_cast(1234), + static_cast(nullptr)); // PR #4221 + }); + // test_accessors m.def("accessor_api", [](const py::object &o) { auto d = py::dict(); @@ -527,6 +554,9 @@ TEST_SUBMODULE(pytypes, m) { m.def("hash_function", [](py::object obj) { return py::hash(std::move(obj)); }); + m.def("obj_contains", + [](py::object &obj, const py::object &key) { return obj.contains(key); }); + m.def("test_number_protocol", [](const py::object &a, const py::object &b) { py::list l; l.append(a.equal(b)); @@ -756,4 +786,38 @@ TEST_SUBMODULE(pytypes, m) { } return o; }); + + // testing immutable object augmented assignment: #issue 3812 + m.def("inplace_append", [](py::object &a, const py::object &b) { + a += b; + return a; + }); + m.def("inplace_subtract", [](py::object &a, const py::object &b) { + a -= b; + return a; + }); + m.def("inplace_multiply", [](py::object &a, const py::object &b) { + a *= b; + return a; + }); + m.def("inplace_divide", [](py::object &a, const py::object &b) { + a /= b; + return a; + }); + m.def("inplace_or", [](py::object &a, const py::object &b) { + a |= b; + return a; + }); + m.def("inplace_and", [](py::object &a, const py::object &b) { + a &= b; + return a; + }); + m.def("inplace_lshift", [](py::object &a, const py::object &b) { + a <<= b; + return a; + }); + m.def("inplace_rshift", [](py::object &a, const py::object &b) { + a >>= b; + return a; + }); } diff --git a/pybind11/tests/test_pytypes.py b/pybind11/tests/test_pytypes.py index 7a0a8b4..eda7a20 100644 --- a/pybind11/tests/test_pytypes.py +++ b/pybind11/tests/test_pytypes.py @@ -9,7 +9,13 @@ from pybind11_tests import pytypes as m -def test_handle_from_move_only_type_with_operator_PyObject(): # noqa: N802 +def test_obj_class_name(): + assert m.obj_class_name(None) == "NoneType" + assert m.obj_class_name(list) == "list" + assert m.obj_class_name([]) == "list" + + +def test_handle_from_move_only_type_with_operator_PyObject(): assert m.handle_from_move_only_type_with_operator_PyObject_ncnst() assert m.handle_from_move_only_type_with_operator_PyObject_const() @@ -26,6 +32,22 @@ def test_iterator(doc): assert doc(m.get_iterator) == "get_iterator() -> Iterator" +@pytest.mark.parametrize( + ("pytype", "from_iter_func"), + [ + (frozenset, m.get_frozenset_from_iterable), + (list, m.get_list_from_iterable), + (set, m.get_set_from_iterable), + (tuple, m.get_tuple_from_iterable), + ], +) +def test_from_iterable(pytype, from_iter_func): + my_iter = iter(range(10)) + s = from_iter_func(my_iter) + assert type(s) == pytype + assert s == pytype(range(10)) + + def test_iterable(doc): assert doc(m.get_iterable) == "get_iterable() -> Iterable" @@ -65,7 +87,7 @@ def test_list(capture, doc): assert doc(m.print_list) == "print_list(arg0: list) -> None" -def test_none(capture, doc): +def test_none(doc): assert doc(m.get_none) == "get_none() -> None" assert doc(m.print_none) == "print_none(arg0: None) -> None" @@ -152,6 +174,31 @@ def test_dict(capture, doc): assert m.dict_keyword_constructor() == {"x": 1, "y": 2, "z": 3} +class CustomContains: + d = {"key": None} + + def __contains__(self, m): + return m in self.d + + +@pytest.mark.parametrize( + ("arg", "func"), + [ + (set(), m.anyset_contains), + ({}, m.dict_contains), + (CustomContains(), m.obj_contains), + ], +) +@pytest.mark.xfail("env.PYPY and sys.pypy_version_info < (7, 3, 10)", strict=False) +def test_unhashable_exceptions(arg, func): + class Unhashable: + __hash__ = None + + with pytest.raises(TypeError) as exc_info: + func(arg, Unhashable()) + assert "unhashable type:" in str(exc_info.value) + + def test_tuple(): assert m.tuple_no_args() == () assert m.tuple_ssize_t() == () @@ -203,6 +250,20 @@ def __repr__(self): m.str_from_string_from_str(ucs_surrogates_str) +@pytest.mark.parametrize( + "func", + [ + m.str_from_bytes_input, + m.str_from_cstr_input, + m.str_from_std_string_input, + ], +) +def test_surrogate_pairs_unicode_error(func): + input_str = "\ud83d\ude4f".encode("utf-8", "surrogatepass") + with pytest.raises(UnicodeDecodeError): + func(input_str) + + def test_bytes(doc): assert m.bytes_from_char_ssize_t().decode() == "green" assert m.bytes_from_char_size_t().decode() == "purple" @@ -212,7 +273,7 @@ def test_bytes(doc): assert doc(m.bytes_from_str) == "bytes_from_str() -> bytes" -def test_bytearray(doc): +def test_bytearray(): assert m.bytearray_from_char_ssize_t().decode() == "$%" assert m.bytearray_from_char_size_t().decode() == "@$!" assert m.bytearray_from_string().decode() == "foo" @@ -258,6 +319,19 @@ def test_capsule(capture): """ ) + with capture: + a = m.return_capsule_with_destructor_3() + del a + pytest.gc_collect() + assert ( + capture.unordered + == """ + creating capsule + destructing capsule: 1233 + original name: oname + """ + ) + with capture: a = m.return_renamed_capsule_with_destructor_2() del a @@ -283,6 +357,17 @@ def test_capsule(capture): """ ) + with capture: + a = m.return_capsule_with_explicit_nullptr_dtor() + del a + pytest.gc_collect() + assert ( + capture.unordered + == """ + creating capsule with explicit nullptr dtor + """ + ) + def test_accessors(): class SubTestObject: @@ -313,7 +398,7 @@ def func(self, x, *args): assert d["implicit_list"] == [1, 2, 3] assert all(x in TestObject.__dict__ for x in d["implicit_dict"]) - assert m.tuple_accessor(tuple()) == (0, 1, 2) + assert m.tuple_accessor(()) == (0, 1, 2) d = m.accessor_assignment() assert d["get"] == 0 @@ -403,7 +488,7 @@ def test_pybind11_str_raw_str(): assert cvt({}) == "{}" assert cvt({3: 4}) == "{3: 4}" assert cvt(set()) == "set()" - assert cvt({3, 3}) == "{3}" + assert cvt({3}) == "{3}" valid_orig = "Η±" valid_utf8 = valid_orig.encode("utf-8") @@ -464,7 +549,7 @@ def test_print(capture): assert str(excinfo.value) == "Unable to convert call argument " + ( "'1' of type 'UnregisteredType' to Python object" if detailed_error_messages_enabled - else "to Python object (#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)" + else "'1' to Python object (#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)" ) @@ -521,7 +606,7 @@ def test_issue2361(): @pytest.mark.parametrize( - "method, args, fmt, expected_view", + ("method", "args", "fmt", "expected_view"), [ (m.test_memoryview_object, (b"red",), "B", b"red"), (m.test_memoryview_buffer_info, (b"green",), "B", b"green"), @@ -579,7 +664,7 @@ def test_memoryview_from_memory(): def test_builtin_functions(): - assert m.get_len([i for i in range(42)]) == 42 + assert m.get_len(list(range(42))) == 42 with pytest.raises(TypeError) as exc_info: m.get_len(i for i in range(42)) assert str(exc_info.value) in [ @@ -623,7 +708,7 @@ def test_pass_bytes_or_unicode_to_string_types(): @pytest.mark.parametrize( - "create_weakref, create_weakref_with_callback", + ("create_weakref", "create_weakref_with_callback"), [ (m.weakref_from_handle, m.weakref_from_handle_and_function), (m.weakref_from_object, m.weakref_from_object_and_function), @@ -638,7 +723,7 @@ class WeaklyReferenced: callback_called = False - def callback(wr): + def callback(_): nonlocal callback_called callback_called = True @@ -658,7 +743,7 @@ def callback(wr): @pytest.mark.parametrize( - "create_weakref, has_callback", + ("create_weakref", "has_callback"), [ (m.weakref_from_handle, False), (m.weakref_from_object, False), @@ -676,10 +761,7 @@ def callback(_): ob = C() # Should raise TypeError on CPython with pytest.raises(TypeError) if not env.PYPY else contextlib.nullcontext(): - if has_callback: - _ = create_weakref(ob, callback) - else: - _ = create_weakref(ob) + _ = create_weakref(ob, callback) if has_callback else create_weakref(ob) def test_cpp_iterators(): @@ -739,3 +821,78 @@ def test_populate_obj_str_attrs(): new_attrs = {k: v for k, v in new_o.__dict__.items() if not k.startswith("_")} assert all(isinstance(v, str) for v in new_attrs.values()) assert len(new_attrs) == pop + + +@pytest.mark.parametrize( + ("a", "b"), + [("foo", "bar"), (1, 2), (1.0, 2.0), (list(range(3)), list(range(3, 6)))], +) +def test_inplace_append(a, b): + expected = a + b + assert m.inplace_append(a, b) == expected + + +@pytest.mark.parametrize( + ("a", "b"), [(3, 2), (3.0, 2.0), (set(range(3)), set(range(2)))] +) +def test_inplace_subtract(a, b): + expected = a - b + assert m.inplace_subtract(a, b) == expected + + +@pytest.mark.parametrize(("a", "b"), [(3, 2), (3.0, 2.0), ([1], 3)]) +def test_inplace_multiply(a, b): + expected = a * b + assert m.inplace_multiply(a, b) == expected + + +@pytest.mark.parametrize(("a", "b"), [(6, 3), (6.0, 3.0)]) +def test_inplace_divide(a, b): + expected = a / b + assert m.inplace_divide(a, b) == expected + + +@pytest.mark.parametrize( + ("a", "b"), + [ + (False, True), + ( + set(), + { + 1, + }, + ), + ], +) +def test_inplace_or(a, b): + expected = a | b + assert m.inplace_or(a, b) == expected + + +@pytest.mark.parametrize( + ("a", "b"), + [ + (True, False), + ( + {1, 2, 3}, + { + 1, + }, + ), + ], +) +def test_inplace_and(a, b): + expected = a & b + assert m.inplace_and(a, b) == expected + + +@pytest.mark.parametrize(("a", "b"), [(8, 1), (-3, 2)]) +def test_inplace_lshift(a, b): + expected = a << b + assert m.inplace_lshift(a, b) == expected + + +@pytest.mark.parametrize(("a", "b"), [(8, 1), (-2, 2)]) +def test_inplace_rshift(a, b): + expected = a >> b + assert m.inplace_rshift(a, b) == expected diff --git a/pybind11/tests/test_sequences_and_iterators.cpp b/pybind11/tests/test_sequences_and_iterators.cpp index b867f49..1de65ed 100644 --- a/pybind11/tests/test_sequences_and_iterators.cpp +++ b/pybind11/tests/test_sequences_and_iterators.cpp @@ -559,4 +559,23 @@ TEST_SUBMODULE(sequences_and_iterators, m) { []() { return py::make_iterator(list); }); m.def("make_iterator_2", []() { return py::make_iterator(list); }); + + // test_iterator on c arrays + // #4100: ensure lvalue required as increment operand + class CArrayHolder { + public: + CArrayHolder(double x, double y, double z) { + values[0] = x; + values[1] = y; + values[2] = z; + }; + double values[3]; + }; + + py::class_(m, "CArrayHolder") + .def(py::init()) + .def( + "__iter__", + [](const CArrayHolder &v) { return py::make_iterator(v.values, v.values + 3); }, + py::keep_alive<0, 1>()); } diff --git a/pybind11/tests/test_sequences_and_iterators.py b/pybind11/tests/test_sequences_and_iterators.py index 062e3b3..dc129f2 100644 --- a/pybind11/tests/test_sequences_and_iterators.py +++ b/pybind11/tests/test_sequences_and_iterators.py @@ -1,5 +1,5 @@ import pytest -from pytest import approx +from pytest import approx # noqa: PT013 from pybind11_tests import ConstructorStats from pybind11_tests import sequences_and_iterators as m @@ -103,7 +103,8 @@ def test_sequence(): assert "Sequence" in repr(s) assert len(s) == 5 - assert s[0] == 0 and s[3] == 0 + assert s[0] == 0 + assert s[3] == 0 assert 12.34 not in s s[0], s[3] = 12.34, 56.78 assert 12.34 in s @@ -241,3 +242,11 @@ def test_iterator_rvp(): assert list(m.make_iterator_1()) == [1, 2, 3] assert list(m.make_iterator_2()) == [1, 2, 3] assert not isinstance(m.make_iterator_1(), type(m.make_iterator_2())) + + +def test_carray_iterator(): + """#4100: Check for proper iterator overload with C-Arrays""" + args_gt = [float(i) for i in range(3)] + arr_h = m.CArrayHolder(*args_gt) + args = list(arr_h) + assert args_gt == args diff --git a/pybind11/tests/test_smart_ptr.cpp b/pybind11/tests/test_smart_ptr.cpp index 2c23a9a..6d9efce 100644 --- a/pybind11/tests/test_smart_ptr.cpp +++ b/pybind11/tests/test_smart_ptr.cpp @@ -266,14 +266,14 @@ struct ElementList { // It is always possible to construct a ref from an Object* pointer without // possible inconsistencies, hence the 'true' argument at the end. // Make pybind11 aware of the non-standard getter member function -namespace pybind11 { +namespace PYBIND11_NAMESPACE { namespace detail { template struct holder_helper> { static const T *get(const ref &p) { return p.get_ptr(); } }; } // namespace detail -} // namespace pybind11 +} // namespace PYBIND11_NAMESPACE // Make pybind aware of the ref-counted wrapper type (s): PYBIND11_DECLARE_HOLDER_TYPE(T, ref, true); diff --git a/pybind11/tests/test_stl.cpp b/pybind11/tests/test_stl.cpp index 38d32fd..d45465d 100644 --- a/pybind11/tests/test_stl.cpp +++ b/pybind11/tests/test_stl.cpp @@ -23,7 +23,7 @@ #if defined(PYBIND11_TEST_BOOST) # include -namespace pybind11 { +namespace PYBIND11_NAMESPACE { namespace detail { template struct type_caster> : optional_caster> {}; @@ -31,7 +31,7 @@ struct type_caster> : optional_caster> {}; template <> struct type_caster : void_caster {}; } // namespace detail -} // namespace pybind11 +} // namespace PYBIND11_NAMESPACE #endif // Test with `std::variant` in C++17 mode, or with `boost::variant` in C++11/14 @@ -43,7 +43,7 @@ using std::variant; # define PYBIND11_TEST_VARIANT 1 using boost::variant; -namespace pybind11 { +namespace PYBIND11_NAMESPACE { namespace detail { template struct type_caster> : variant_caster> {}; @@ -56,7 +56,7 @@ struct visit_helper { } }; } // namespace detail -} // namespace pybind11 +} // namespace PYBIND11_NAMESPACE #endif PYBIND11_MAKE_OPAQUE(std::vector>); @@ -159,13 +159,13 @@ class ReferenceSensitiveOptional { std::vector storage; }; -namespace pybind11 { +namespace PYBIND11_NAMESPACE { namespace detail { template struct type_caster> : optional_caster> {}; } // namespace detail -} // namespace pybind11 +} // namespace PYBIND11_NAMESPACE TEST_SUBMODULE(stl, m) { // test_vector @@ -176,9 +176,14 @@ TEST_SUBMODULE(stl, m) { m.def("load_bool_vector", [](const std::vector &v) { return v.at(0) == true && v.at(1) == false; }); // Unnumbered regression (caused by #936): pointers to stl containers aren't castable - static std::vector lvv{2}; m.def( - "cast_ptr_vector", []() { return &lvv; }, py::return_value_policy::reference); + "cast_ptr_vector", + []() { + // Using no-destructor idiom to side-step warnings from overzealous compilers. + static auto *v = new std::vector{2}; + return v; + }, + py::return_value_policy::reference); // test_deque m.def("cast_deque", []() { return std::deque{1}; }); @@ -237,6 +242,7 @@ TEST_SUBMODULE(stl, m) { lvn["b"].emplace_back(); // add a list lvn["b"].back().emplace_back(); // add an array lvn["b"].back().emplace_back(); // add another array + static std::vector lvv{2}; m.def("cast_lv_vector", []() -> const decltype(lvv) & { return lvv; }); m.def("cast_lv_array", []() -> const decltype(lva) & { return lva; }); m.def("cast_lv_map", []() -> const decltype(lvm) & { return lvm; }); diff --git a/pybind11/tests/test_stl.py b/pybind11/tests/test_stl.py index d30c382..8a614f8 100644 --- a/pybind11/tests/test_stl.py +++ b/pybind11/tests/test_stl.py @@ -14,7 +14,7 @@ def test_vector(doc): assert m.cast_bool_vector() == [True, False] assert m.load_bool_vector([True, False]) - assert m.load_bool_vector(tuple([True, False])) + assert m.load_bool_vector((True, False)) assert doc(m.cast_vector) == "cast_vector() -> List[int]" assert doc(m.load_vector) == "load_vector(arg0: List[int]) -> bool" @@ -23,7 +23,7 @@ def test_vector(doc): assert m.cast_ptr_vector() == ["lvalue", "lvalue"] -def test_deque(doc): +def test_deque(): """std::deque <-> list""" lst = m.cast_deque() assert lst == [1] @@ -39,8 +39,11 @@ def test_array(doc): assert m.load_array(lst) assert m.load_array(tuple(lst)) - assert doc(m.cast_array) == "cast_array() -> List[int[2]]" - assert doc(m.load_array) == "load_array(arg0: List[int[2]]) -> bool" + assert doc(m.cast_array) == "cast_array() -> Annotated[List[int], FixedSize(2)]" + assert ( + doc(m.load_array) + == "load_array(arg0: Annotated[List[int], FixedSize(2)]) -> bool" + ) def test_valarray(doc): @@ -95,7 +98,8 @@ def test_recursive_casting(): # Issue #853 test case: z = m.cast_unique_ptr_vector() - assert z[0].value == 7 and z[1].value == 42 + assert z[0].value == 7 + assert z[1].value == 42 def test_move_out_container(): @@ -366,7 +370,7 @@ def test_issue_1561(): """check fix for issue #1561""" bar = m.Issue1561Outer() bar.list = [m.Issue1561Inner("bar")] - bar.list + assert bar.list assert bar.list[0].data == "bar" diff --git a/pybind11/tests/test_stl_binders.cpp b/pybind11/tests/test_stl_binders.cpp index ca9630b..1681760 100644 --- a/pybind11/tests/test_stl_binders.cpp +++ b/pybind11/tests/test_stl_binders.cpp @@ -70,6 +70,44 @@ NestMap *times_hundred(int n) { return m; } +/* + * Recursive data structures as test for issue #4623 + */ +struct RecursiveVector : std::vector { + using Parent = std::vector; + using Parent::Parent; +}; + +struct RecursiveMap : std::map { + using Parent = std::map; + using Parent::Parent; +}; + +/* + * Pybind11 does not catch more complicated recursion schemes, such as mutual + * recursion. + * In that case custom recursive_container_traits specializations need to be added, + * thus manually telling pybind11 about the recursion. + */ +struct MutuallyRecursiveContainerPairMV; +struct MutuallyRecursiveContainerPairVM; + +struct MutuallyRecursiveContainerPairMV : std::map {}; +struct MutuallyRecursiveContainerPairVM : std::vector {}; + +namespace pybind11 { +namespace detail { +template +struct recursive_container_traits { + using type_to_check_recursively = recursive_bottom; +}; +template +struct recursive_container_traits { + using type_to_check_recursively = recursive_bottom; +}; +} // namespace detail +} // namespace pybind11 + TEST_SUBMODULE(stl_binders, m) { // test_vector_int py::bind_vector>(m, "VectorInt", py::buffer_protocol()); @@ -129,6 +167,12 @@ TEST_SUBMODULE(stl_binders, m) { m, "VectorUndeclStruct", py::buffer_protocol()); }); + // Bind recursive container types + py::bind_vector(m, "RecursiveVector"); + py::bind_map(m, "RecursiveMap"); + py::bind_map(m, "MutuallyRecursiveContainerPairMV"); + py::bind_vector(m, "MutuallyRecursiveContainerPairVM"); + // The rest depends on numpy: try { py::module_::import("numpy"); diff --git a/pybind11/tests/test_stl_binders.py b/pybind11/tests/test_stl_binders.py index d5e9ccc..e002f5b 100644 --- a/pybind11/tests/test_stl_binders.py +++ b/pybind11/tests/test_stl_binders.py @@ -186,9 +186,9 @@ def test_map_string_double(): um["ua"] = 1.1 um["ub"] = 2.6 - assert sorted(list(um)) == ["ua", "ub"] + assert sorted(um) == ["ua", "ub"] assert list(um.keys()) == list(um) - assert sorted(list(um.items())) == [("ua", 1.1), ("ub", 2.6)] + assert sorted(um.items()) == [("ua", 1.1), ("ub", 2.6)] assert list(zip(um.keys(), um.values())) == list(um.items()) assert "UnorderedMapStringDouble" in str(um) @@ -304,8 +304,52 @@ def test_map_delitem(): um["ua"] = 1.1 um["ub"] = 2.6 - assert sorted(list(um)) == ["ua", "ub"] - assert sorted(list(um.items())) == [("ua", 1.1), ("ub", 2.6)] + assert sorted(um) == ["ua", "ub"] + assert sorted(um.items()) == [("ua", 1.1), ("ub", 2.6)] del um["ua"] - assert sorted(list(um)) == ["ub"] - assert sorted(list(um.items())) == [("ub", 2.6)] + assert sorted(um) == ["ub"] + assert sorted(um.items()) == [("ub", 2.6)] + + +def test_map_view_types(): + map_string_double = m.MapStringDouble() + unordered_map_string_double = m.UnorderedMapStringDouble() + map_string_double_const = m.MapStringDoubleConst() + unordered_map_string_double_const = m.UnorderedMapStringDoubleConst() + + assert map_string_double.keys().__class__.__name__ == "KeysView[str]" + assert map_string_double.values().__class__.__name__ == "ValuesView[float]" + assert map_string_double.items().__class__.__name__ == "ItemsView[str, float]" + + keys_type = type(map_string_double.keys()) + assert type(unordered_map_string_double.keys()) is keys_type + assert type(map_string_double_const.keys()) is keys_type + assert type(unordered_map_string_double_const.keys()) is keys_type + + values_type = type(map_string_double.values()) + assert type(unordered_map_string_double.values()) is values_type + assert type(map_string_double_const.values()) is values_type + assert type(unordered_map_string_double_const.values()) is values_type + + items_type = type(map_string_double.items()) + assert type(unordered_map_string_double.items()) is items_type + assert type(map_string_double_const.items()) is items_type + assert type(unordered_map_string_double_const.items()) is items_type + + +def test_recursive_vector(): + recursive_vector = m.RecursiveVector() + recursive_vector.append(m.RecursiveVector()) + recursive_vector[0].append(m.RecursiveVector()) + recursive_vector[0].append(m.RecursiveVector()) + # Can't use len() since test_stl_binders.cpp does not include stl.h, + # so the necessary conversion is missing + assert recursive_vector[0].count(m.RecursiveVector()) == 2 + + +def test_recursive_map(): + recursive_map = m.RecursiveMap() + recursive_map[100] = m.RecursiveMap() + recursive_map[100][101] = m.RecursiveMap() + recursive_map[100][102] = m.RecursiveMap() + assert list(recursive_map[100].keys()) == [101, 102] diff --git a/pybind11/tests/test_tagbased_polymorphic.cpp b/pybind11/tests/test_tagbased_polymorphic.cpp index 2807e86..12ba653 100644 --- a/pybind11/tests/test_tagbased_polymorphic.cpp +++ b/pybind11/tests/test_tagbased_polymorphic.cpp @@ -117,7 +117,7 @@ std::string Animal::name_of_kind(Kind kind) { return raw_name; } -namespace pybind11 { +namespace PYBIND11_NAMESPACE { template struct polymorphic_type_hook::value>> { static const void *get(const itype *src, const std::type_info *&type) { @@ -125,7 +125,7 @@ struct polymorphic_type_hook(m, "Animal").def_readonly("name", &Animal::name); diff --git a/pybind11/tests/test_type_caster_pyobject_ptr.cpp b/pybind11/tests/test_type_caster_pyobject_ptr.cpp new file mode 100644 index 0000000..1667ea1 --- /dev/null +++ b/pybind11/tests/test_type_caster_pyobject_ptr.cpp @@ -0,0 +1,130 @@ +#include +#include +#include + +#include "pybind11_tests.h" + +#include +#include + +namespace { + +std::vector make_vector_pyobject_ptr(const py::object &ValueHolder) { + std::vector vec_obj; + for (int i = 1; i < 3; i++) { + vec_obj.push_back(ValueHolder(i * 93).release().ptr()); + } + // This vector now owns the refcounts. + return vec_obj; +} + +} // namespace + +TEST_SUBMODULE(type_caster_pyobject_ptr, m) { + m.def("cast_from_pyobject_ptr", []() { + PyObject *ptr = PyLong_FromLongLong(6758L); + return py::cast(ptr, py::return_value_policy::take_ownership); + }); + m.def("cast_handle_to_pyobject_ptr", [](py::handle obj) { + auto rc1 = obj.ref_count(); + auto *ptr = py::cast(obj); + auto rc2 = obj.ref_count(); + if (rc2 != rc1 + 1) { + return -1; + } + return 100 - py::reinterpret_steal(ptr).attr("value").cast(); + }); + m.def("cast_object_to_pyobject_ptr", [](py::object obj) { + py::handle hdl = obj; + auto rc1 = hdl.ref_count(); + auto *ptr = py::cast(std::move(obj)); + auto rc2 = hdl.ref_count(); + if (rc2 != rc1) { + return -1; + } + return 300 - py::reinterpret_steal(ptr).attr("value").cast(); + }); + m.def("cast_list_to_pyobject_ptr", [](py::list lst) { + // This is to cover types implicitly convertible to object. + py::handle hdl = lst; + auto rc1 = hdl.ref_count(); + auto *ptr = py::cast(std::move(lst)); + auto rc2 = hdl.ref_count(); + if (rc2 != rc1) { + return -1; + } + return 400 - static_cast(py::len(py::reinterpret_steal(ptr))); + }); + + m.def( + "return_pyobject_ptr", + []() { return PyLong_FromLongLong(2314L); }, + py::return_value_policy::take_ownership); + m.def("pass_pyobject_ptr", [](PyObject *ptr) { + return 200 - py::reinterpret_borrow(ptr).attr("value").cast(); + }); + + m.def("call_callback_with_object_return", + [](const std::function &cb, int value) { return cb(value); }); + m.def( + "call_callback_with_pyobject_ptr_return", + [](const std::function &cb, int value) { return cb(value); }, + py::return_value_policy::take_ownership); + m.def( + "call_callback_with_pyobject_ptr_arg", + [](const std::function &cb, py::handle obj) { return cb(obj.ptr()); }, + py::arg("cb"), // This triggers return_value_policy::automatic_reference + py::arg("obj")); + + m.def("cast_to_pyobject_ptr_nullptr", [](bool set_error) { + if (set_error) { + PyErr_SetString(PyExc_RuntimeError, "Reflective of healthy error handling."); + } + PyObject *ptr = nullptr; + py::cast(ptr); + }); + + m.def("cast_to_pyobject_ptr_non_nullptr_with_error_set", []() { + PyErr_SetString(PyExc_RuntimeError, "Reflective of unhealthy error handling."); + py::cast(Py_None); + }); + + m.def("pass_list_pyobject_ptr", [](const std::vector &vec_obj) { + int acc = 0; + for (const auto &ptr : vec_obj) { + acc = acc * 1000 + py::reinterpret_borrow(ptr).attr("value").cast(); + } + return acc; + }); + + m.def("return_list_pyobject_ptr_take_ownership", + make_vector_pyobject_ptr, + // Ownership is transferred one-by-one when the vector is converted to a Python list. + py::return_value_policy::take_ownership); + + m.def("return_list_pyobject_ptr_reference", + make_vector_pyobject_ptr, + // Ownership is not transferred. + py::return_value_policy::reference); + + m.def("dec_ref_each_pyobject_ptr", [](const std::vector &vec_obj) { + std::size_t i = 0; + for (; i < vec_obj.size(); i++) { + py::handle h(vec_obj[i]); + if (static_cast(h.ref_count()) < 2) { + break; // Something is badly wrong. + } + h.dec_ref(); + } + return i; + }); + + m.def("pass_pyobject_ptr_and_int", [](PyObject *, int) {}); + +#ifdef PYBIND11_NO_COMPILE_SECTION // Change to ifndef for manual testing. + { + PyObject *ptr = nullptr; + (void) py::cast(*ptr); + } +#endif +} diff --git a/pybind11/tests/test_type_caster_pyobject_ptr.py b/pybind11/tests/test_type_caster_pyobject_ptr.py new file mode 100644 index 0000000..1f1ece2 --- /dev/null +++ b/pybind11/tests/test_type_caster_pyobject_ptr.py @@ -0,0 +1,104 @@ +import pytest + +from pybind11_tests import type_caster_pyobject_ptr as m + + +# For use as a temporary user-defined object, to maximize sensitivity of the tests below. +class ValueHolder: + def __init__(self, value): + self.value = value + + +def test_cast_from_pyobject_ptr(): + assert m.cast_from_pyobject_ptr() == 6758 + + +def test_cast_handle_to_pyobject_ptr(): + assert m.cast_handle_to_pyobject_ptr(ValueHolder(24)) == 76 + + +def test_cast_object_to_pyobject_ptr(): + assert m.cast_object_to_pyobject_ptr(ValueHolder(43)) == 257 + + +def test_cast_list_to_pyobject_ptr(): + assert m.cast_list_to_pyobject_ptr([1, 2, 3, 4, 5]) == 395 + + +def test_return_pyobject_ptr(): + assert m.return_pyobject_ptr() == 2314 + + +def test_pass_pyobject_ptr(): + assert m.pass_pyobject_ptr(ValueHolder(82)) == 118 + + +@pytest.mark.parametrize( + "call_callback", + [ + m.call_callback_with_object_return, + m.call_callback_with_pyobject_ptr_return, + ], +) +def test_call_callback_with_object_return(call_callback): + def cb(value): + if value < 0: + raise ValueError("Raised from cb") + return ValueHolder(1000 - value) + + assert call_callback(cb, 287).value == 713 + + with pytest.raises(ValueError, match="^Raised from cb$"): + call_callback(cb, -1) + + +def test_call_callback_with_pyobject_ptr_arg(): + def cb(obj): + return 300 - obj.value + + assert m.call_callback_with_pyobject_ptr_arg(cb, ValueHolder(39)) == 261 + + +@pytest.mark.parametrize("set_error", [True, False]) +def test_cast_to_python_nullptr(set_error): + expected = { + True: r"^Reflective of healthy error handling\.$", + False: ( + r"^Internal error: pybind11::error_already_set called " + r"while Python error indicator not set\.$" + ), + }[set_error] + with pytest.raises(RuntimeError, match=expected): + m.cast_to_pyobject_ptr_nullptr(set_error) + + +def test_cast_to_python_non_nullptr_with_error_set(): + with pytest.raises(SystemError) as excinfo: + m.cast_to_pyobject_ptr_non_nullptr_with_error_set() + assert str(excinfo.value) == "src != nullptr but PyErr_Occurred()" + assert str(excinfo.value.__cause__) == "Reflective of unhealthy error handling." + + +def test_pass_list_pyobject_ptr(): + acc = m.pass_list_pyobject_ptr([ValueHolder(842), ValueHolder(452)]) + assert acc == 842452 + + +def test_return_list_pyobject_ptr_take_ownership(): + vec_obj = m.return_list_pyobject_ptr_take_ownership(ValueHolder) + assert [e.value for e in vec_obj] == [93, 186] + + +def test_return_list_pyobject_ptr_reference(): + vec_obj = m.return_list_pyobject_ptr_reference(ValueHolder) + assert [e.value for e in vec_obj] == [93, 186] + # Commenting out the next `assert` will leak the Python references. + # An easy way to see evidence of the leaks: + # Insert `while True:` as the first line of this function and monitor the + # process RES (Resident Memory Size) with the Unix top command. + assert m.dec_ref_each_pyobject_ptr(vec_obj) == 2 + + +def test_type_caster_name_via_incompatible_function_arguments_type_error(): + with pytest.raises(TypeError, match=r"1\. \(arg0: object, arg1: int\) -> None"): + m.pass_pyobject_ptr_and_int(ValueHolder(101), ValueHolder(202)) diff --git a/pybind11/tests/test_unnamed_namespace_a.cpp b/pybind11/tests/test_unnamed_namespace_a.cpp new file mode 100644 index 0000000..2152e64 --- /dev/null +++ b/pybind11/tests/test_unnamed_namespace_a.cpp @@ -0,0 +1,38 @@ +#include "pybind11_tests.h" + +namespace { +struct any_struct {}; +} // namespace + +TEST_SUBMODULE(unnamed_namespace_a, m) { + if (py::detail::get_type_info(typeid(any_struct)) == nullptr) { + py::class_(m, "unnamed_namespace_a_any_struct"); + } else { + m.attr("unnamed_namespace_a_any_struct") = py::none(); + } + m.attr("PYBIND11_INTERNALS_VERSION") = PYBIND11_INTERNALS_VERSION; + m.attr("defined_WIN32_or__WIN32") = +#if defined(WIN32) || defined(_WIN32) + true; +#else + false; +#endif + m.attr("defined___clang__") = +#if defined(__clang__) + true; +#else + false; +#endif + m.attr("defined__LIBCPP_VERSION") = +#if defined(_LIBCPP_VERSION) + true; +#else + false; +#endif + m.attr("defined___GLIBCXX__") = +#if defined(__GLIBCXX__) + true; +#else + false; +#endif +} diff --git a/pybind11/tests/test_unnamed_namespace_a.py b/pybind11/tests/test_unnamed_namespace_a.py new file mode 100644 index 0000000..9d9856c --- /dev/null +++ b/pybind11/tests/test_unnamed_namespace_a.py @@ -0,0 +1,34 @@ +import pytest + +from pybind11_tests import unnamed_namespace_a as m +from pybind11_tests import unnamed_namespace_b as mb + +XFAIL_CONDITION = ( + "(m.PYBIND11_INTERNALS_VERSION <= 4 and (m.defined___clang__ or not m.defined___GLIBCXX__))" + " or " + "(m.PYBIND11_INTERNALS_VERSION >= 5 and not m.defined_WIN32_or__WIN32" + " and " + "(m.defined___clang__ or m.defined__LIBCPP_VERSION))" +) +XFAIL_REASON = "Known issues: https://github.com/pybind/pybind11/pull/4319" + + +@pytest.mark.xfail(XFAIL_CONDITION, reason=XFAIL_REASON, strict=False) +@pytest.mark.parametrize( + "any_struct", [m.unnamed_namespace_a_any_struct, mb.unnamed_namespace_b_any_struct] +) +def test_have_class_any_struct(any_struct): + assert any_struct is not None + + +def test_have_at_least_one_class_any_struct(): + assert ( + m.unnamed_namespace_a_any_struct is not None + or mb.unnamed_namespace_b_any_struct is not None + ) + + +@pytest.mark.xfail(XFAIL_CONDITION, reason=XFAIL_REASON, strict=True) +def test_have_both_class_any_struct(): + assert m.unnamed_namespace_a_any_struct is not None + assert mb.unnamed_namespace_b_any_struct is not None diff --git a/pybind11/tests/test_unnamed_namespace_b.cpp b/pybind11/tests/test_unnamed_namespace_b.cpp new file mode 100644 index 0000000..f97757a --- /dev/null +++ b/pybind11/tests/test_unnamed_namespace_b.cpp @@ -0,0 +1,13 @@ +#include "pybind11_tests.h" + +namespace { +struct any_struct {}; +} // namespace + +TEST_SUBMODULE(unnamed_namespace_b, m) { + if (py::detail::get_type_info(typeid(any_struct)) == nullptr) { + py::class_(m, "unnamed_namespace_b_any_struct"); + } else { + m.attr("unnamed_namespace_b_any_struct") = py::none(); + } +} diff --git a/pybind11/tests/test_unnamed_namespace_b.py b/pybind11/tests/test_unnamed_namespace_b.py new file mode 100644 index 0000000..4bcaa7a --- /dev/null +++ b/pybind11/tests/test_unnamed_namespace_b.py @@ -0,0 +1,5 @@ +from pybind11_tests import unnamed_namespace_b as m + + +def test_have_attr_any_struct(): + assert hasattr(m, "unnamed_namespace_b_any_struct") diff --git a/pybind11/tests/test_vector_unique_ptr_member.cpp b/pybind11/tests/test_vector_unique_ptr_member.cpp new file mode 100644 index 0000000..680cf9a --- /dev/null +++ b/pybind11/tests/test_vector_unique_ptr_member.cpp @@ -0,0 +1,54 @@ +#include "pybind11_tests.h" + +#include +#include +#include + +namespace pybind11_tests { +namespace vector_unique_ptr_member { + +struct DataType {}; + +// Reduced from a use case in the wild. +struct VectorOwner { + static std::unique_ptr Create(std::size_t num_elems) { + return std::unique_ptr( + new VectorOwner(std::vector>(num_elems))); + } + + std::size_t data_size() const { return data_.size(); } + +private: + explicit VectorOwner(std::vector> data) : data_(std::move(data)) {} + + const std::vector> data_; +}; + +} // namespace vector_unique_ptr_member +} // namespace pybind11_tests + +namespace pybind11 { +namespace detail { + +template <> +struct is_copy_constructible + : std::false_type {}; + +template <> +struct is_move_constructible + : std::false_type {}; + +} // namespace detail +} // namespace pybind11 + +using namespace pybind11_tests::vector_unique_ptr_member; + +py::object py_cast_VectorOwner_ptr(VectorOwner *ptr) { return py::cast(ptr); } + +TEST_SUBMODULE(vector_unique_ptr_member, m) { + py::class_(m, "VectorOwner") + .def_static("Create", &VectorOwner::Create) + .def("data_size", &VectorOwner::data_size); + + m.def("py_cast_VectorOwner_ptr", py_cast_VectorOwner_ptr); +} diff --git a/pybind11/tests/test_vector_unique_ptr_member.py b/pybind11/tests/test_vector_unique_ptr_member.py new file mode 100644 index 0000000..2da3d97 --- /dev/null +++ b/pybind11/tests/test_vector_unique_ptr_member.py @@ -0,0 +1,14 @@ +import pytest + +from pybind11_tests import vector_unique_ptr_member as m + + +@pytest.mark.parametrize("num_elems", range(3)) +def test_create(num_elems): + vo = m.VectorOwner.Create(num_elems) + assert vo.data_size() == num_elems + + +def test_cast(): + vo = m.VectorOwner.Create(0) + assert m.py_cast_VectorOwner_ptr(vo) is vo diff --git a/pybind11/tests/test_virtual_functions.cpp b/pybind11/tests/test_virtual_functions.cpp index 323aa0d..93b136a 100644 --- a/pybind11/tests/test_virtual_functions.cpp +++ b/pybind11/tests/test_virtual_functions.cpp @@ -173,7 +173,8 @@ struct AdderBase { using DataVisitor = std::function; virtual void - operator()(const Data &first, const Data &second, const DataVisitor &visitor) const = 0; + operator()(const Data &first, const Data &second, const DataVisitor &visitor) const + = 0; virtual ~AdderBase() = default; AdderBase() = default; AdderBase(const AdderBase &) = delete; diff --git a/pybind11/tests/test_virtual_functions.py b/pybind11/tests/test_virtual_functions.py index 4d00d36..c17af7d 100644 --- a/pybind11/tests/test_virtual_functions.py +++ b/pybind11/tests/test_virtual_functions.py @@ -192,8 +192,7 @@ def test_move_support(): class NCVirtExt(m.NCVirt): def get_noncopyable(self, a, b): # Constructs and returns a new instance: - nc = m.NonCopyable(a * a, b * b) - return nc + return m.NonCopyable(a * a, b * b) def get_movable(self, a, b): # Return a referenced copy @@ -256,7 +255,7 @@ def dispatch(self): assert m.dispatch_issue_go(b) == "Yay.." -def test_recursive_dispatch_issue(msg): +def test_recursive_dispatch_issue(): """#3357: Recursive dispatch fails to find python function override""" class Data(m.Data): @@ -269,7 +268,7 @@ def __call__(self, first, second, visitor): # lambda is a workaround, which adds extra frame to the # current CPython thread. Removing lambda reveals the bug # [https://github.com/pybind/pybind11/issues/3357] - (lambda: visitor(Data(first.value + second.value)))() + (lambda: visitor(Data(first.value + second.value)))() # noqa: PLC3002 class StoreResultVisitor: def __init__(self): diff --git a/pybind11/tools/FindCatch.cmake b/pybind11/tools/FindCatch.cmake index 57bba58..5d3fcbf 100644 --- a/pybind11/tools/FindCatch.cmake +++ b/pybind11/tools/FindCatch.cmake @@ -36,10 +36,14 @@ endfunction() function(_download_catch version destination_dir) message(STATUS "Downloading catch v${version}...") set(url https://github.com/philsquared/Catch/releases/download/v${version}/catch.hpp) - file(DOWNLOAD ${url} "${destination_dir}/catch.hpp" STATUS status) + file( + DOWNLOAD ${url} "${destination_dir}/catch.hpp" + STATUS status + LOG log) list(GET status 0 error) if(error) - message(FATAL_ERROR "Could not download ${url}") + string(REPLACE "\n" "\n " log " ${log}") + message(FATAL_ERROR "Could not download URL:\n" " ${url}\n" "Log:\n" "${log}") endif() set(CATCH_INCLUDE_DIR "${destination_dir}" diff --git a/pybind11/tools/FindPythonLibsNew.cmake b/pybind11/tools/FindPythonLibsNew.cmake index a5a628f..ce558d4 100644 --- a/pybind11/tools/FindPythonLibsNew.cmake +++ b/pybind11/tools/FindPythonLibsNew.cmake @@ -151,9 +151,13 @@ if(NOT _PYTHON_SUCCESS MATCHES 0) return() endif() +option( + PYBIND11_PYTHONLIBS_OVERWRITE + "Overwrite cached values read from Python library (classic search). Turn off if cross-compiling and manually setting these values." + ON) # Can manually set values when cross-compiling macro(_PYBIND11_GET_IF_UNDEF lst index name) - if(NOT DEFINED "${name}") + if(PYBIND11_PYTHONLIBS_OVERWRITE OR NOT DEFINED "${name}") list(GET "${lst}" "${index}" "${name}") endif() endmacro() @@ -204,7 +208,9 @@ string(REGEX REPLACE "\\\\" "/" PYTHON_PREFIX "${PYTHON_PREFIX}") string(REGEX REPLACE "\\\\" "/" PYTHON_INCLUDE_DIR "${PYTHON_INCLUDE_DIR}") string(REGEX REPLACE "\\\\" "/" PYTHON_SITE_PACKAGES "${PYTHON_SITE_PACKAGES}") -if(CMAKE_HOST_WIN32) +if(DEFINED PYTHON_LIBRARY) + # Don't write to PYTHON_LIBRARY if it's already set +elseif(CMAKE_HOST_WIN32) set(PYTHON_LIBRARY "${PYTHON_PREFIX}/libs/python${PYTHON_LIBRARY_SUFFIX}.lib") # when run in a venv, PYTHON_PREFIX points to it. But the libraries remain in the @@ -270,7 +276,7 @@ if(NOT PYTHON_DEBUG_LIBRARY) endif() set(PYTHON_DEBUG_LIBRARIES "${PYTHON_DEBUG_LIBRARY}") -find_package_message(PYTHON "Found PythonLibs: ${PYTHON_LIBRARY}" +find_package_message(PYTHON "Found PythonLibs: ${PYTHON_LIBRARIES}" "${PYTHON_EXECUTABLE}${PYTHON_VERSION_STRING}") set(PYTHONLIBS_FOUND TRUE) diff --git a/pybind11/tools/JoinPaths.cmake b/pybind11/tools/JoinPaths.cmake new file mode 100644 index 0000000..c68d91b --- /dev/null +++ b/pybind11/tools/JoinPaths.cmake @@ -0,0 +1,23 @@ +# This module provides function for joining paths +# known from most languages +# +# SPDX-License-Identifier: (MIT OR CC0-1.0) +# Copyright 2020 Jan Tojnar +# https://github.com/jtojnar/cmake-snips +# +# Modelled after Python’s os.path.join +# https://docs.python.org/3.7/library/os.path.html#os.path.join +# Windows not supported +function(join_paths joined_path first_path_segment) + set(temp_path "${first_path_segment}") + foreach(current_segment IN LISTS ARGN) + if(NOT ("${current_segment}" STREQUAL "")) + if(IS_ABSOLUTE "${current_segment}") + set(temp_path "${current_segment}") + else() + set(temp_path "${temp_path}/${current_segment}") + endif() + endif() + endforeach() + set(${joined_path} "${temp_path}" PARENT_SCOPE) +endfunction() diff --git a/pybind11/tools/codespell_ignore_lines_from_errors.py b/pybind11/tools/codespell_ignore_lines_from_errors.py new file mode 100644 index 0000000..4ec9add --- /dev/null +++ b/pybind11/tools/codespell_ignore_lines_from_errors.py @@ -0,0 +1,39 @@ +"""Simple script for rebuilding .codespell-ignore-lines + +Usage: + +cat < /dev/null > .codespell-ignore-lines +pre-commit run --all-files codespell >& /tmp/codespell_errors.txt +python3 tools/codespell_ignore_lines_from_errors.py /tmp/codespell_errors.txt > .codespell-ignore-lines + +git diff to review changes, then commit, push. +""" + +import sys +from typing import List + + +def run(args: List[str]) -> None: + assert len(args) == 1, "codespell_errors.txt" + cache = {} + done = set() + with open(args[0]) as f: + lines = f.read().splitlines() + + for line in sorted(lines): + i = line.find(" ==> ") + if i > 0: + flds = line[:i].split(":") + if len(flds) >= 2: + filename, line_num = flds[:2] + if filename not in cache: + with open(filename) as f: + cache[filename] = f.read().splitlines() + supp = cache[filename][int(line_num) - 1] + if supp not in done: + print(supp) + done.add(supp) + + +if __name__ == "__main__": + run(args=sys.argv[1:]) diff --git a/pybind11/tools/make_changelog.py b/pybind11/tools/make_changelog.py index 839040a..b5bd832 100755 --- a/pybind11/tools/make_changelog.py +++ b/pybind11/tools/make_changelog.py @@ -31,8 +31,10 @@ missing = [] for issue in issues: - changelog = ENTRY.findall(issue.body) - if changelog: + changelog = ENTRY.findall(issue.body or "") + if not changelog or not changelog[0]: + missing.append(issue) + else: (msg,) = changelog if not msg.startswith("* "): msg = "* " + msg @@ -44,9 +46,6 @@ print(Syntax(msg, "rst", theme="ansi_light", word_wrap=True)) print() - else: - missing.append(issue) - if missing: print() print("[blue]" + "-" * 30) diff --git a/pybind11/tools/pybind11.pc.in b/pybind11/tools/pybind11.pc.in new file mode 100644 index 0000000..402f0b3 --- /dev/null +++ b/pybind11/tools/pybind11.pc.in @@ -0,0 +1,7 @@ +prefix=@prefix_for_pc_file@ +includedir=@includedir_for_pc_file@ + +Name: @PROJECT_NAME@ +Description: Seamless operability between C++11 and Python +Version: @PROJECT_VERSION@ +Cflags: -I${includedir} diff --git a/pybind11/tools/pybind11Common.cmake b/pybind11/tools/pybind11Common.cmake index e1fb601..308d1b7 100644 --- a/pybind11/tools/pybind11Common.cmake +++ b/pybind11/tools/pybind11Common.cmake @@ -5,8 +5,8 @@ Adds the following targets:: pybind11::pybind11 - link to headers and pybind11 pybind11::module - Adds module links pybind11::embed - Adds embed links - pybind11::lto - Link time optimizations (manual selection) - pybind11::thin_lto - Link time optimizations (manual selection) + pybind11::lto - Link time optimizations (only if CMAKE_INTERPROCEDURAL_OPTIMIZATION is not set) + pybind11::thin_lto - Link time optimizations (only if CMAKE_INTERPROCEDURAL_OPTIMIZATION is not set) pybind11::python_link_helper - Adds link to Python libraries pybind11::windows_extras - MSVC bigobj and mp for building multithreaded pybind11::opt_size - avoid optimizations that increase code size @@ -20,7 +20,7 @@ Adds the following functions:: # CMake 3.10 has an include_guard command, but we can't use that yet # include_guard(global) (pre-CMake 3.10) -if(TARGET pybind11::lto) +if(TARGET pybind11::pybind11) return() endif() @@ -163,11 +163,19 @@ endif() # --------------------- Python specifics ------------------------- +# CMake 3.27 removes the classic FindPythonInterp if CMP0148 is NEW +if(CMAKE_VERSION VERSION_LESS "3.27") + set(_pybind11_missing_old_python "OLD") +else() + cmake_policy(GET CMP0148 _pybind11_missing_old_python) +endif() + # Check to see which Python mode we are in, new, old, or no python if(PYBIND11_NOPYTHON) set(_pybind11_nopython ON) elseif( - PYBIND11_FINDPYTHON + _pybind11_missing_old_python STREQUAL "NEW" + OR PYBIND11_FINDPYTHON OR Python_FOUND OR Python2_FOUND OR Python3_FOUND) @@ -311,6 +319,16 @@ function(_pybind11_generate_lto target prefer_thin_lto) HAS_FLTO "-flto${cxx_append}" "-flto${linker_append}" PYBIND11_LTO_CXX_FLAGS PYBIND11_LTO_LINKER_FLAGS) endif() + elseif(CMAKE_CXX_COMPILER_ID MATCHES "IntelLLVM") + # IntelLLVM equivalent to LTO is called IPO; also IntelLLVM is WIN32/UNIX + # WARNING/HELP WANTED: This block of code is currently not covered by pybind11 GitHub Actions! + if(WIN32) + _pybind11_return_if_cxx_and_linker_flags_work( + HAS_INTEL_IPO "-Qipo" "-Qipo" PYBIND11_LTO_CXX_FLAGS PYBIND11_LTO_LINKER_FLAGS) + else() + _pybind11_return_if_cxx_and_linker_flags_work( + HAS_INTEL_IPO "-ipo" "-ipo" PYBIND11_LTO_CXX_FLAGS PYBIND11_LTO_LINKER_FLAGS) + endif() elseif(CMAKE_CXX_COMPILER_ID MATCHES "Intel") # Intel equivalent to LTO is called IPO _pybind11_return_if_cxx_and_linker_flags_work(HAS_INTEL_IPO "-ipo" "-ipo" @@ -362,11 +380,13 @@ function(_pybind11_generate_lto target prefer_thin_lto) endif() endfunction() -add_library(pybind11::lto IMPORTED INTERFACE ${optional_global}) -_pybind11_generate_lto(pybind11::lto FALSE) +if(NOT DEFINED CMAKE_INTERPROCEDURAL_OPTIMIZATION) + add_library(pybind11::lto IMPORTED INTERFACE ${optional_global}) + _pybind11_generate_lto(pybind11::lto FALSE) -add_library(pybind11::thin_lto IMPORTED INTERFACE ${optional_global}) -_pybind11_generate_lto(pybind11::thin_lto TRUE) + add_library(pybind11::thin_lto IMPORTED INTERFACE ${optional_global}) + _pybind11_generate_lto(pybind11::thin_lto TRUE) +endif() # ---------------------- pybind11_strip ----------------------------- diff --git a/pybind11/tools/pybind11Config.cmake.in b/pybind11/tools/pybind11Config.cmake.in index 9383e8c..5734f43 100644 --- a/pybind11/tools/pybind11Config.cmake.in +++ b/pybind11/tools/pybind11Config.cmake.in @@ -63,7 +63,9 @@ Modes There are two modes provided; classic, which is built on the old Python discovery packages in CMake, or the new FindPython mode, which uses FindPython -from 3.12+ forward (3.15+ _highly_ recommended). +from 3.12+ forward (3.15+ _highly_ recommended). If you set the minimum or +maximum version of CMake to 3.27+, then FindPython is the default (since +FindPythonInterp/FindPythonLibs has been removed via policy `CMP0148`). New FindPython mode ^^^^^^^^^^^^^^^^^^^ diff --git a/pybind11/tools/pybind11NewTools.cmake b/pybind11/tools/pybind11NewTools.cmake index abba0fe..7d7424a 100644 --- a/pybind11/tools/pybind11NewTools.cmake +++ b/pybind11/tools/pybind11NewTools.cmake @@ -9,7 +9,7 @@ if(CMAKE_VERSION VERSION_LESS 3.12) message(FATAL_ERROR "You cannot use the new FindPython module with CMake < 3.12") endif() -include_guard(GLOBAL) +include_guard(DIRECTORY) get_property( is_config @@ -233,7 +233,9 @@ function(pybind11_add_module target_name) endif() endif() - if(NOT MSVC AND NOT ${CMAKE_BUILD_TYPE} MATCHES Debug|RelWithDebInfo) + # Use case-insensitive comparison to match the result of $ + string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE) + if(NOT MSVC AND NOT "${uppercase_CMAKE_BUILD_TYPE}" MATCHES DEBUG|RELWITHDEBINFO) # Strip unnecessary sections of the binary on Linux/macOS pybind11_strip(${target_name}) endif() diff --git a/pybind11/tools/pybind11Tools.cmake b/pybind11/tools/pybind11Tools.cmake index 5535e87..66ad00a 100644 --- a/pybind11/tools/pybind11Tools.cmake +++ b/pybind11/tools/pybind11Tools.cmake @@ -115,6 +115,7 @@ if(PYTHON_IS_DEBUG) PROPERTY INTERFACE_COMPILE_DEFINITIONS Py_DEBUG) endif() +# The <3.11 code here does not support release/debug builds at the same time, like on vcpkg if(CMAKE_VERSION VERSION_LESS 3.11) set_property( TARGET pybind11::module @@ -130,16 +131,19 @@ if(CMAKE_VERSION VERSION_LESS 3.11) APPEND PROPERTY INTERFACE_LINK_LIBRARIES pybind11::pybind11 $) else() + # The IMPORTED INTERFACE library here is to ensure that "debug" and "release" get processed outside + # of a generator expression - https://gitlab.kitware.com/cmake/cmake/-/issues/18424, as they are + # target_link_library keywords rather than real libraries. + add_library(pybind11::_ClassicPythonLibraries IMPORTED INTERFACE) + target_link_libraries(pybind11::_ClassicPythonLibraries INTERFACE ${PYTHON_LIBRARIES}) target_link_libraries( pybind11::module INTERFACE pybind11::python_link_helper - "$<$,$>:$>" - ) + "$<$,$>:pybind11::_ClassicPythonLibraries>") target_link_libraries(pybind11::embed INTERFACE pybind11::pybind11 - $) - + pybind11::_ClassicPythonLibraries) endif() function(pybind11_extension name) @@ -208,7 +212,9 @@ function(pybind11_add_module target_name) endif() endif() - if(NOT MSVC AND NOT ${CMAKE_BUILD_TYPE} MATCHES Debug|RelWithDebInfo) + # Use case-insensitive comparison to match the result of $ + string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE) + if(NOT MSVC AND NOT "${uppercase_CMAKE_BUILD_TYPE}" MATCHES DEBUG|RELWITHDEBINFO) pybind11_strip(${target_name}) endif() diff --git a/pybind11/tools/setup_global.py.in b/pybind11/tools/setup_global.py.in index 8aa3871..885ac5c 100644 --- a/pybind11/tools/setup_global.py.in +++ b/pybind11/tools/setup_global.py.in @@ -27,9 +27,11 @@ class InstallHeadersNested(install_headers): main_headers = glob.glob("pybind11/include/pybind11/*.h") detail_headers = glob.glob("pybind11/include/pybind11/detail/*.h") +eigen_headers = glob.glob("pybind11/include/pybind11/eigen/*.h") stl_headers = glob.glob("pybind11/include/pybind11/stl/*.h") cmake_files = glob.glob("pybind11/share/cmake/pybind11/*.cmake") -headers = main_headers + detail_headers + stl_headers +pkgconfig_files = glob.glob("pybind11/share/pkgconfig/*.pc") +headers = main_headers + detail_headers + stl_headers + eigen_headers cmdclass = {"install_headers": InstallHeadersNested} $extra_cmd @@ -51,8 +53,10 @@ setup( headers=headers, data_files=[ (base + "share/cmake/pybind11", cmake_files), + (base + "share/pkgconfig", pkgconfig_files), (base + "include/pybind11", main_headers), (base + "include/pybind11/detail", detail_headers), + (base + "include/pybind11/eigen", eigen_headers), (base + "include/pybind11/stl", stl_headers), ], cmdclass=cmdclass, diff --git a/pybind11/tools/setup_main.py.in b/pybind11/tools/setup_main.py.in index 738d73f..6358cc7 100644 --- a/pybind11/tools/setup_main.py.in +++ b/pybind11/tools/setup_main.py.in @@ -15,15 +15,19 @@ setup( "pybind11", "pybind11.include.pybind11", "pybind11.include.pybind11.detail", + "pybind11.include.pybind11.eigen", "pybind11.include.pybind11.stl", "pybind11.share.cmake.pybind11", + "pybind11.share.pkgconfig", ], package_data={ "pybind11": ["py.typed"], "pybind11.include.pybind11": ["*.h"], "pybind11.include.pybind11.detail": ["*.h"], + "pybind11.include.pybind11.eigen": ["*.h"], "pybind11.include.pybind11.stl": ["*.h"], "pybind11.share.cmake.pybind11": ["*.cmake"], + "pybind11.share.pkgconfig": ["*.pc"], }, extras_require={ "global": ["pybind11_global==$version"]