From a00c6a88d3b909a37f782573eb11ba489ed2199c Mon Sep 17 00:00:00 2001 From: Andrej Dundovic Date: Fri, 12 Jan 2024 08:06:00 +0100 Subject: [PATCH] update pybind11, cmakelists with official cmake python3 --- CMakeLists.txt | 14 +- Dockerfile | 11 +- lib/pybind11/.pre-commit-config.yaml | 143 ++++----- lib/pybind11/.readthedocs.yml | 21 +- lib/pybind11/CMakeLists.txt | 57 +++- lib/pybind11/MANIFEST.in | 3 +- lib/pybind11/README.rst | 2 +- lib/pybind11/SECURITY.md | 13 + lib/pybind11/include/pybind11/attr.h | 18 +- lib/pybind11/include/pybind11/buffer_info.h | 17 +- lib/pybind11/include/pybind11/cast.h | 121 ++++++-- lib/pybind11/include/pybind11/detail/class.h | 41 ++- lib/pybind11/include/pybind11/detail/common.h | 133 +++++++-- lib/pybind11/include/pybind11/detail/descr.h | 13 + lib/pybind11/include/pybind11/detail/init.h | 42 +-- .../include/pybind11/detail/internals.h | 74 ++++- .../pybind11/detail/type_caster_base.h | 274 +++++++++++++++--- lib/pybind11/include/pybind11/eigen/common.h | 9 + lib/pybind11/include/pybind11/eigen/matrix.h | 44 +-- lib/pybind11/include/pybind11/eigen/tensor.h | 44 ++- lib/pybind11/include/pybind11/embed.h | 133 ++++++--- lib/pybind11/include/pybind11/gil.h | 18 +- .../include/pybind11/gil_safe_call_once.h | 91 ++++++ lib/pybind11/include/pybind11/numpy.h | 89 ++++-- lib/pybind11/include/pybind11/options.h | 16 + lib/pybind11/include/pybind11/pybind11.h | 256 +++++++++------- lib/pybind11/include/pybind11/pytypes.h | 199 ++++++++++--- lib/pybind11/include/pybind11/stl.h | 19 +- lib/pybind11/include/pybind11/stl_bind.h | 179 ++++++++---- .../pybind11/type_caster_pyobject_ptr.h | 61 ++++ lib/pybind11/include/pybind11/typing.h | 117 ++++++++ lib/pybind11/noxfile.py | 2 +- lib/pybind11/pybind11/__init__.py | 2 +- lib/pybind11/pybind11/__main__.py | 8 +- lib/pybind11/pybind11/_version.py | 2 +- lib/pybind11/pybind11/commands.py | 2 +- lib/pybind11/pybind11/setup_helpers.py | 24 +- lib/pybind11/pyproject.toml | 48 ++- lib/pybind11/setup.cfg | 9 +- lib/pybind11/setup.py | 2 +- lib/pybind11/tools/FindCatch.cmake | 8 +- lib/pybind11/tools/FindPythonLibsNew.cmake | 31 +- .../codespell_ignore_lines_from_errors.py | 8 +- lib/pybind11/tools/make_changelog.py | 37 ++- lib/pybind11/tools/pybind11Common.cmake | 68 +++-- lib/pybind11/tools/pybind11Config.cmake.in | 6 +- lib/pybind11/tools/pybind11NewTools.cmake | 101 +++++-- lib/pybind11/tools/pybind11Tools.cmake | 20 +- 48 files changed, 1975 insertions(+), 675 deletions(-) create mode 100644 lib/pybind11/SECURITY.md create mode 100644 lib/pybind11/include/pybind11/eigen/common.h create mode 100644 lib/pybind11/include/pybind11/gil_safe_call_once.h create mode 100644 lib/pybind11/include/pybind11/type_caster_pyobject_ptr.h create mode 100644 lib/pybind11/include/pybind11/typing.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 641e8faf..1de6300c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -345,12 +345,12 @@ target_link_libraries(hermes ${HERMES_EXTRA_LIBRARIES}) # Python # ---------------------------------------------------------------------------- option(ENABLE_PYTHON "Create python library via Pybind11" OFF) -include(FindPython) -find_package(Python COMPONENTS Interpreter Development) +include(FindPython3) +find_package(Python3 COMPONENTS Interpreter Development) -if(ENABLE_PYTHON AND PYTHONLIBS_FOUND) - include_directories(${PYTHON_INCLUDE_DIRS}) - set(PYBIND11_PYTHON_VERSION ${PYTHON_DOT_VERSION}) +if(ENABLE_PYTHON AND Python3_Development_FOUND) + include_directories(${Python3_INCLUDE_DIRS}) + set(PYBIND11_PYTHON_VERSION ${Python3_VERSION}) add_subdirectory(lib/pybind11) # for PYBIND11_EXPAND_SIDE_EFFECTS in pybind11.h - temporary, # should be removed after the fix lands to the upstream @@ -381,8 +381,8 @@ if(ENABLE_PYTHON AND PYTHONLIBS_FOUND) LINKER_LANGUAGE CXX) target_link_libraries(pyhermes PRIVATE hermes) - install(TARGETS pyhermes LIBRARY DESTINATION "${PYTHON_SITE_PACKAGES}") -endif(ENABLE_PYTHON AND PYTHONLIBS_FOUND) + install(TARGETS pyhermes LIBRARY DESTINATION "${Python3_SITELIB}") +endif(ENABLE_PYTHON AND Python3_Development_FOUND) # ------------------------------------------------------------------ # Documentation diff --git a/Dockerfile b/Dockerfile index 93697e05..2dfe2fc6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,6 +23,7 @@ USER ${NB_UID} # Install Python 3 packages RUN mamba install --yes \ + 'conda-forge::sphinx' \ 'healpy' && \ mamba clean --all -f -y && \ fix-permissions "${CONDA_DIR}" && \ @@ -32,13 +33,15 @@ RUN mamba install --yes \ WORKDIR /tmp RUN git clone https://github.com/cosmicrays/hermes.git WORKDIR /tmp/hermes -RUN conda init +ENV PATH /opt/conda/bin:$PATH +RUN /bin/bash -c "source activate base" RUN mkdir build && \ cd build && \ CMAKE_PREFIX_PATH=${CONDA_DIR} cmake \ - -DPython_EXECUTABLE=${CONDA_DIR}/bin/${PYTHON_VERSION} \ - -DPython_INCLUDE_DIRS=${CONDA_DIR}/include/${PYTHON_VERSION} \ - -DPython_LIBRARIES=${CONDA_DIR}/lib/${PYTHON_VERSION} \ + -DPython3_EXECUTABLE=${CONDA_DIR}/bin/python${PYTHON_VERSION} \ + -DPython3_INCLUDE_DIRS=${CONDA_DIR}/include/python${PYTHON_VERSION} \ + -DPython3_LIBRARIES=${CONDA_DIR}/lib/python${PYTHON_VERSION} \ + -DENABLE_PYTHON=On \ -DCMAKE_INSTALL_PREFIX=${CONDA_DIR} \ -DENABLE_TESTING=On .. && \ make -j && \ diff --git a/lib/pybind11/.pre-commit-config.yaml b/lib/pybind11/.pre-commit-config.yaml index fd079dd0..cb28515d 100644 --- a/lib/pybind11/.pre-commit-config.yaml +++ b/lib/pybind11/.pre-commit-config.yaml @@ -22,9 +22,47 @@ ci: exclude: ^tools/JoinPaths.cmake$ repos: + +# Clang format the codebase automatically +- repo: https://github.com/pre-commit/mirrors-clang-format + rev: "v17.0.6" + hooks: + - id: clang-format + types_or: [c++, c, cuda] + +# Ruff, the Python auto-correcting linter/formatter written in Rust +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.1.9 + hooks: + - id: ruff + args: ["--fix", "--show-fixes"] + - id: ruff-format + +# Check static types with mypy +- repo: https://github.com/pre-commit/mirrors-mypy + rev: "v1.8.0" + hooks: + - id: mypy + args: [] + exclude: ^(tests|docs)/ + additional_dependencies: + - markdown-it-py<3 # Drop this together with dropping Python 3.7 support. + - nox + - rich + - types-setuptools + +# CMake formatting +- repo: https://github.com/cheshirekow/cmake-format-precommit + rev: "v0.6.13" + hooks: + - id: cmake-format + additional_dependencies: [pyyaml] + types: [file] + files: (\.cmake|CMakeLists.txt)(.in)?$ + # Standard hooks - repo: https://github.com/pre-commit/pre-commit-hooks - rev: "v4.3.0" + rev: "v4.5.0" hooks: - id: check-added-large-files - id: check-case-conflict @@ -39,109 +77,38 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace -# Upgrade old Python syntax -- repo: https://github.com/asottile/pyupgrade - rev: "v3.2.0" - hooks: - - id: pyupgrade - args: [--py36-plus] - -# Nicely sort includes -- repo: https://github.com/PyCQA/isort - rev: "5.10.1" - hooks: - - id: isort - -# Black, the code formatter, natively supports pre-commit -- repo: https://github.com/psf/black - rev: "22.10.0" # Keep in sync with blacken-docs - hooks: - - id: black - # Also code format the docs - repo: https://github.com/asottile/blacken-docs - rev: "v1.12.1" + rev: "1.16.0" hooks: - id: blacken-docs additional_dependencies: - - black==22.10.0 # keep in sync with black hook + - black==23.* # Changes tabs to spaces - repo: https://github.com/Lucas-C/pre-commit-hooks - rev: "v1.3.1" + rev: "v1.5.4" hooks: - id: remove-tabs +# Avoid directional quotes - repo: https://github.com/sirosen/texthooks - rev: "0.4.0" + rev: "0.6.3" hooks: - id: fix-ligatures - id: fix-smartquotes -# Autoremoves unused imports -- repo: https://github.com/hadialqattan/pycln - rev: "v2.1.1" - hooks: - - id: pycln - stages: [manual] - # Checking for common mistakes - repo: https://github.com/pre-commit/pygrep-hooks - rev: "v1.9.0" + rev: "v1.10.0" hooks: - - id: python-check-blanket-noqa - - id: python-check-blanket-type-ignore - - id: python-no-log-warn - - id: python-use-type-annotations - id: rst-backticks - id: rst-directive-colons - id: rst-inline-touching-normal -# Automatically remove noqa that are not used -- repo: https://github.com/asottile/yesqa - rev: "v1.4.0" - hooks: - - id: yesqa - additional_dependencies: &flake8_dependencies - - flake8-bugbear - - pep8-naming - -# Flake8 also supports pre-commit natively (same author) -- repo: https://github.com/PyCQA/flake8 - rev: "5.0.4" - hooks: - - id: flake8 - exclude: ^(docs/.*|tools/.*)$ - additional_dependencies: *flake8_dependencies - -# PyLint has native support - not always usable, but works for us -- repo: https://github.com/PyCQA/pylint - rev: "v2.15.5" - hooks: - - id: pylint - files: ^pybind11 - -# CMake formatting -- repo: https://github.com/cheshirekow/cmake-format-precommit - rev: "v0.6.13" - hooks: - - id: cmake-format - additional_dependencies: [pyyaml] - types: [file] - files: (\.cmake|CMakeLists.txt)(.in)?$ - -# Check static types with mypy -- repo: https://github.com/pre-commit/mirrors-mypy - rev: "v0.982" - hooks: - - id: mypy - args: [] - exclude: ^(tests|docs)/ - additional_dependencies: [nox, rich] - # Checks the manifest for missing files (native support) - repo: https://github.com/mgedmin/check-manifest - rev: "0.48" + rev: "0.49" hooks: - id: check-manifest # This is a slow hook, so only run this if --hook-stage manual is passed @@ -152,15 +119,15 @@ repos: # Use tools/codespell_ignore_lines_from_errors.py # to rebuild .codespell-ignore-lines - repo: https://github.com/codespell-project/codespell - rev: "v2.2.2" + rev: "v2.2.6" hooks: - id: codespell exclude: ".supp$" - args: ["-x", ".codespell-ignore-lines"] + args: ["-x.codespell-ignore-lines", "-Lccompiler"] # Check for common shell mistakes - repo: https://github.com/shellcheck-py/shellcheck-py - rev: "v0.8.0.4" + rev: "v0.9.0.6" hooks: - id: shellcheck @@ -170,12 +137,12 @@ repos: - id: disallow-caps name: Disallow improper capitalization language: pygrep - entry: PyBind|Numpy|Cmake|CCache|PyTest + entry: PyBind|\bNumpy\b|Cmake|CCache|PyTest exclude: ^\.pre-commit-config.yaml$ -# Clang format the codebase automatically -- repo: https://github.com/pre-commit/mirrors-clang-format - rev: "v14.0.6" +# PyLint has native support - not always usable, but works for us +- repo: https://github.com/PyCQA/pylint + rev: "v3.0.3" hooks: - - id: clang-format - types_or: [c++, c, cuda] + - id: pylint + files: ^pybind11 diff --git a/lib/pybind11/.readthedocs.yml b/lib/pybind11/.readthedocs.yml index c9c61617..a2b802f7 100644 --- a/lib/pybind11/.readthedocs.yml +++ b/lib/pybind11/.readthedocs.yml @@ -1,3 +1,20 @@ +# https://blog.readthedocs.com/migrate-configuration-v2/ + +version: 2 + +build: + os: ubuntu-22.04 + apt_packages: + - librsvg2-bin + tools: + python: "3.11" + +sphinx: + configuration: docs/conf.py + python: - version: 3 -requirements_file: docs/requirements.txt + install: + - requirements: docs/requirements.txt + +formats: + - pdf diff --git a/lib/pybind11/CMakeLists.txt b/lib/pybind11/CMakeLists.txt index 0d932038..1e75e99e 100644 --- a/lib/pybind11/CMakeLists.txt +++ b/lib/pybind11/CMakeLists.txt @@ -5,15 +5,25 @@ # All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. -cmake_minimum_required(VERSION 3.4) +# Propagate this policy (FindPythonInterp removal) so it can be detected later +if(NOT CMAKE_VERSION VERSION_LESS "3.27") + cmake_policy(GET CMP0148 _pybind11_cmp0148) +endif() + +cmake_minimum_required(VERSION 3.5) -# The `cmake_minimum_required(VERSION 3.4...3.22)` syntax does not work with +# The `cmake_minimum_required(VERSION 3.5...3.27)` syntax does not work with # some versions of VS that have a patched CMake 3.11. This forces us to emulate # the behavior using the following workaround: -if(${CMAKE_VERSION} VERSION_LESS 3.22) +if(${CMAKE_VERSION} VERSION_LESS 3.27) cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}) else() - cmake_policy(VERSION 3.22) + cmake_policy(VERSION 3.27) +endif() + +if(_pybind11_cmp0148) + cmake_policy(SET CMP0148 ${_pybind11_cmp0148}) + unset(_pybind11_cmp0148) endif() # Avoid infinite recursion if tests include this as a subdirectory @@ -82,9 +92,15 @@ if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR) set(pybind11_system "") set_property(GLOBAL PROPERTY USE_FOLDERS ON) + if(CMAKE_VERSION VERSION_LESS "3.18") + set(_pybind11_findpython_default OFF) + else() + set(_pybind11_findpython_default ON) + endif() else() set(PYBIND11_MASTER_PROJECT OFF) set(pybind11_system SYSTEM) + set(_pybind11_findpython_default OFF) endif() # Options @@ -106,9 +122,18 @@ cmake_dependent_option( "Install pybind11 headers in Python include directory instead of default installation prefix" OFF "PYBIND11_INSTALL" OFF) -cmake_dependent_option(PYBIND11_FINDPYTHON "Force new FindPython" OFF +cmake_dependent_option(PYBIND11_FINDPYTHON "Force new FindPython" ${_pybind11_findpython_default} "NOT CMAKE_VERSION VERSION_LESS 3.12" OFF) +# Allow PYTHON_EXECUTABLE if in FINDPYTHON mode and building pybind11's tests +# (makes transition easier while we support both modes). +if(PYBIND11_MASTER_PROJECT + AND PYBIND11_FINDPYTHON + AND DEFINED PYTHON_EXECUTABLE + AND NOT DEFINED Python_EXECUTABLE) + set(Python_EXECUTABLE "${PYTHON_EXECUTABLE}") +endif() + # NB: when adding a header don't forget to also add it to setup.py set(PYBIND11_HEADERS include/pybind11/detail/class.h @@ -126,11 +151,13 @@ set(PYBIND11_HEADERS include/pybind11/complex.h include/pybind11/options.h include/pybind11/eigen.h + include/pybind11/eigen/common.h include/pybind11/eigen/matrix.h include/pybind11/eigen/tensor.h include/pybind11/embed.h include/pybind11/eval.h include/pybind11/gil.h + include/pybind11/gil_safe_call_once.h include/pybind11/iostream.h include/pybind11/functional.h include/pybind11/numpy.h @@ -139,7 +166,9 @@ set(PYBIND11_HEADERS include/pybind11/pytypes.h include/pybind11/stl.h include/pybind11/stl_bind.h - include/pybind11/stl/filesystem.h) + include/pybind11/stl/filesystem.h + include/pybind11/type_caster_pyobject_ptr.h + include/pybind11/typing.h) # Compare with grep and warn if mismatched if(PYBIND11_MASTER_PROJECT AND NOT CMAKE_VERSION VERSION_LESS 3.12) @@ -275,7 +304,21 @@ if(PYBIND11_INSTALL) # pkg-config support if(NOT prefix_for_pc_file) - set(prefix_for_pc_file "${CMAKE_INSTALL_PREFIX}") + if(IS_ABSOLUTE "${CMAKE_INSTALL_DATAROOTDIR}") + set(prefix_for_pc_file "${CMAKE_INSTALL_PREFIX}") + else() + set(pc_datarootdir "${CMAKE_INSTALL_DATAROOTDIR}") + if(CMAKE_VERSION VERSION_LESS 3.20) + set(prefix_for_pc_file "\${pcfiledir}/..") + while(pc_datarootdir) + get_filename_component(pc_datarootdir "${pc_datarootdir}" DIRECTORY) + string(APPEND prefix_for_pc_file "/..") + endwhile() + else() + cmake_path(RELATIVE_PATH CMAKE_INSTALL_PREFIX BASE_DIRECTORY CMAKE_INSTALL_DATAROOTDIR + OUTPUT_VARIABLE prefix_for_pc_file) + endif() + endif() endif() join_paths(includedir_for_pc_file "\${prefix}" "${CMAKE_INSTALL_INCLUDEDIR}") configure_file("${CMAKE_CURRENT_SOURCE_DIR}/tools/pybind11.pc.in" diff --git a/lib/pybind11/MANIFEST.in b/lib/pybind11/MANIFEST.in index 033303a7..7ce83c55 100644 --- a/lib/pybind11/MANIFEST.in +++ b/lib/pybind11/MANIFEST.in @@ -1,5 +1,6 @@ +prune tests recursive-include pybind11/include/pybind11 *.h recursive-include pybind11 *.py recursive-include pybind11 py.typed include pybind11/share/cmake/pybind11/*.cmake -include LICENSE README.rst pyproject.toml setup.py setup.cfg +include LICENSE README.rst SECURITY.md pyproject.toml setup.py setup.cfg diff --git a/lib/pybind11/README.rst b/lib/pybind11/README.rst index 3c75edb5..80213a40 100644 --- a/lib/pybind11/README.rst +++ b/lib/pybind11/README.rst @@ -135,7 +135,7 @@ This project was created by `Wenzel Jakob `_. Significant features and/or improvements to the code were contributed by Jonas Adler, Lori A. Burns, Sylvain Corlay, Eric Cousineau, Aaron Gokaslan, Ralf Grosse-Kunstleve, Trent Houliston, Axel -Huebl, @hulucc, Yannick Jadoul, Sergey Lyskov Johan Mabille, Tomasz Miąsko, +Huebl, @hulucc, Yannick Jadoul, Sergey Lyskov, Johan Mabille, Tomasz Miąsko, Dean Moldovan, Ben Pritchard, Jason Rhinelander, Boris Schäling, Pim Schellart, Henry Schreiner, Ivan Smirnov, Boris Staletic, and Patrick Stewart. diff --git a/lib/pybind11/SECURITY.md b/lib/pybind11/SECURITY.md new file mode 100644 index 00000000..3d74611f --- /dev/null +++ b/lib/pybind11/SECURITY.md @@ -0,0 +1,13 @@ +# Security Policy + +## Supported Versions + +Security updates are applied only to the latest release. + +## Reporting a Vulnerability + +If you have discovered a security vulnerability in this project, please report it privately. **Do not disclose it as a public issue.** This gives us time to work with you to fix the issue before public exposure, reducing the chance that the exploit will be used before a patch is released. + +Please disclose it at [security advisory](https://github.com/pybind/pybind11/security/advisories/new). + +This project is maintained by a team of volunteers on a reasonable-effort basis. As such, please give us at least 90 days to work on a fix before public exposure. diff --git a/lib/pybind11/include/pybind11/attr.h b/lib/pybind11/include/pybind11/attr.h index db7cd8ef..1044db94 100644 --- a/lib/pybind11/include/pybind11/attr.h +++ b/lib/pybind11/include/pybind11/attr.h @@ -26,6 +26,9 @@ struct is_method { explicit is_method(const handle &c) : class_(c) {} }; +/// Annotation for setters +struct is_setter {}; + /// Annotation for operators struct is_operator {}; @@ -188,8 +191,8 @@ struct argument_record { struct function_record { function_record() : is_constructor(false), is_new_style_constructor(false), is_stateless(false), - is_operator(false), is_method(false), has_args(false), has_kwargs(false), - prepend(false) {} + is_operator(false), is_method(false), is_setter(false), has_args(false), + has_kwargs(false), prepend(false) {} /// Function name char *name = nullptr; /* why no C++ strings? They generate heavier code.. */ @@ -230,6 +233,9 @@ struct function_record { /// True if this is a method bool is_method : 1; + /// True if this is a setter + bool is_setter : 1; + /// True if the function has a '*args' argument bool has_args : 1; @@ -399,7 +405,7 @@ struct process_attribute : process_attribute_default { template <> struct process_attribute : process_attribute_default { static void init(const char *d, function_record *r) { r->doc = const_cast(d); } - static void init(const char *d, type_record *r) { r->doc = const_cast(d); } + static void init(const char *d, type_record *r) { r->doc = d; } }; template <> struct process_attribute : process_attribute {}; @@ -426,6 +432,12 @@ struct process_attribute : process_attribute_default { } }; +/// Process an attribute which indicates that this function is a setter +template <> +struct process_attribute : process_attribute_default { + static void init(const is_setter &, function_record *r) { r->is_setter = true; } +}; + /// Process an attribute which indicates the parent scope of a method template <> struct process_attribute : process_attribute_default { diff --git a/lib/pybind11/include/pybind11/buffer_info.h b/lib/pybind11/include/pybind11/buffer_info.h index 06120d55..b99ee8be 100644 --- a/lib/pybind11/include/pybind11/buffer_info.h +++ b/lib/pybind11/include/pybind11/buffer_info.h @@ -37,6 +37,9 @@ inline std::vector f_strides(const std::vector &shape, ssize_t return strides; } +template +struct compare_buffer_info; + PYBIND11_NAMESPACE_END(detail) /// Information record describing a Python buffer object @@ -150,6 +153,17 @@ struct buffer_info { Py_buffer *view() const { return m_view; } Py_buffer *&view() { return m_view; } + /* True if the buffer item type is equivalent to `T`. */ + // To define "equivalent" by example: + // `buffer_info::item_type_is_equivalent_to(b)` and + // `buffer_info::item_type_is_equivalent_to(b)` may both be true + // on some platforms, but `int` and `unsigned` will never be equivalent. + // For the ground truth, please inspect `detail::compare_buffer_info<>`. + template + bool item_type_is_equivalent_to() const { + return detail::compare_buffer_info::compare(*this); + } + private: struct private_ctr_tag {}; @@ -170,9 +184,10 @@ struct buffer_info { PYBIND11_NAMESPACE_BEGIN(detail) -template +template struct compare_buffer_info { static bool compare(const buffer_info &b) { + // NOLINTNEXTLINE(bugprone-sizeof-expression) Needed for `PyObject *` return b.format == format_descriptor::format() && b.itemsize == (ssize_t) sizeof(T); } }; diff --git a/lib/pybind11/include/pybind11/cast.h b/lib/pybind11/include/pybind11/cast.h index 430c62f3..8b5beb0e 100644 --- a/lib/pybind11/include/pybind11/cast.h +++ b/lib/pybind11/include/pybind11/cast.h @@ -29,6 +29,9 @@ #include PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +PYBIND11_WARNING_DISABLE_MSVC(4127) + PYBIND11_NAMESPACE_BEGIN(detail) template @@ -39,13 +42,15 @@ using make_caster = type_caster>; // Shortcut for calling a caster's `cast_op_type` cast operator for casting a type_caster to a T template typename make_caster::template cast_op_type cast_op(make_caster &caster) { - return caster.operator typename make_caster::template cast_op_type(); + using result_t = typename make_caster::template cast_op_type; // See PR #4893 + return caster.operator result_t(); } template typename make_caster::template cast_op_type::type> cast_op(make_caster &&caster) { - return std::move(caster).operator typename make_caster:: - template cast_op_type::type>(); + using result_t = typename make_caster::template cast_op_type< + typename std::add_rvalue_reference::type>; // See PR #4893 + return std::move(caster).operator result_t(); } template @@ -88,7 +93,8 @@ public: template >::value, \ - int> = 0> \ + int> \ + = 0> \ static ::pybind11::handle cast( \ T_ *src, ::pybind11::return_value_policy policy, ::pybind11::handle parent) { \ if (!src) \ @@ -389,7 +395,7 @@ struct string_caster { // For UTF-8 we avoid the need for a temporary `bytes` object by using // `PyUnicode_AsUTF8AndSize`. - if (PYBIND11_SILENCE_MSVC_C4127(UTF_N == 8)) { + if (UTF_N == 8) { Py_ssize_t size = -1; const auto *buffer = reinterpret_cast(PyUnicode_AsUTF8AndSize(load_src.ptr(), &size)); @@ -416,7 +422,7 @@ struct string_caster { = reinterpret_cast(PYBIND11_BYTES_AS_STRING(utfNbytes.ptr())); size_t length = (size_t) PYBIND11_BYTES_SIZE(utfNbytes.ptr()) / sizeof(CharT); // Skip BOM for UTF-16/32 - if (PYBIND11_SILENCE_MSVC_C4127(UTF_N > 8)) { + if (UTF_N > 8) { buffer++; length--; } @@ -572,7 +578,7 @@ struct type_caster::value>> { // figure out how long the first encoded character is in bytes to distinguish between these // two errors. We also allow want to allow unicode characters U+0080 through U+00FF, as // those can fit into a single char value. - if (PYBIND11_SILENCE_MSVC_C4127(StringCaster::UTF_N == 8) && str_len > 1 && str_len <= 4) { + if (StringCaster::UTF_N == 8 && str_len > 1 && str_len <= 4) { auto v0 = static_cast(value[0]); // low bits only: 0-127 // 0b110xxxxx - start of 2-byte sequence @@ -598,7 +604,7 @@ struct type_caster::value>> { // UTF-16 is much easier: we can only have a surrogate pair for values above U+FFFF, thus a // surrogate pair with total length 2 instantly indicates a range error (but not a "your // string was too long" error). - else if (PYBIND11_SILENCE_MSVC_C4127(StringCaster::UTF_N == 16) && str_len == 2) { + else if (StringCaster::UTF_N == 16 && str_len == 2) { one_char = static_cast(value[0]); if (one_char >= 0xD800 && one_char < 0xE000) { throw value_error("Character code point not in range(0x10000)"); @@ -657,7 +663,7 @@ class tuple_caster { } static constexpr auto name - = const_name("Tuple[") + concat(make_caster::name...) + const_name("]"); + = const_name("tuple[") + concat(make_caster::name...) + const_name("]"); template using cast_op_type = type; @@ -878,6 +884,10 @@ struct handle_type_name { static constexpr auto name = const_name(PYBIND11_BYTES_NAME); }; template <> +struct handle_type_name { + static constexpr auto name = const_name("Buffer"); +}; +template <> struct handle_type_name { static constexpr auto name = const_name("int"); }; @@ -894,10 +904,22 @@ struct handle_type_name { static constexpr auto name = const_name("float"); }; template <> +struct handle_type_name { + static constexpr auto name = const_name("Callable"); +}; +template <> +struct handle_type_name { + static constexpr auto name = handle_type_name::name; +}; +template <> struct handle_type_name { static constexpr auto name = const_name("None"); }; template <> +struct handle_type_name { + static constexpr auto name = const_name("Sequence"); +}; +template <> struct handle_type_name { static constexpr auto name = const_name("*args"); }; @@ -960,7 +982,7 @@ struct move_always< enable_if_t< all_of, negation>, - std::is_move_constructible, + is_move_constructible, std::is_same>().operator T &()), T &>>::value>> : std::true_type {}; template @@ -971,7 +993,7 @@ struct move_if_unreferenced< enable_if_t< all_of, negation>, - std::is_move_constructible, + is_move_constructible, std::is_same>().operator T &()), T &>>::value>> : std::true_type {}; template @@ -1013,11 +1035,14 @@ type_caster &load_type(type_caster &conv, const handle &ha "Internal error: type_caster should only be used for C++ types"); if (!conv.load(handle, true)) { #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) - throw cast_error("Unable to cast Python instance to C++ type (#define " - "PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); + throw cast_error( + "Unable to cast Python instance of type " + + str(type::handle_of(handle)).cast() + + " to C++ type '?' (#define " + "PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); #else throw cast_error("Unable to cast Python instance of type " - + (std::string) str(type::handle_of(handle)) + " to C++ type '" + + str(type::handle_of(handle)).cast() + " to C++ type '" + type_id() + "'"); #endif } @@ -1034,7 +1059,11 @@ make_caster load_type(const handle &handle) { PYBIND11_NAMESPACE_END(detail) // pytype -> C++ type -template ::value, int> = 0> +template ::value + && !detail::is_same_ignoring_cvref::value, + int> + = 0> T cast(const handle &handle) { using namespace detail; static_assert(!cast_is_temporary_value_reference::value, @@ -1048,6 +1077,34 @@ T cast(const handle &handle) { return T(reinterpret_borrow(handle)); } +// Note that `cast(obj)` increments the reference count of `obj`. +// This is necessary for the case that `obj` is a temporary, and could +// not possibly be different, given +// 1. the established convention that the passed `handle` is borrowed, and +// 2. we don't want to force all generic code using `cast()` to special-case +// handling of `T` = `PyObject *` (to increment the reference count there). +// It is the responsibility of the caller to ensure that the reference count +// is decremented. +template ::value + && detail::is_same_ignoring_cvref::value, + int> + = 0> +T cast(Handle &&handle) { + return handle.inc_ref().ptr(); +} +// To optimize way an inc_ref/dec_ref cycle: +template ::value + && detail::is_same_ignoring_cvref::value, + int> + = 0> +T cast(Object &&obj) { + return obj.release().ptr(); +} + // C++ type -> py::object template ::value, int> = 0> object cast(T &&value, @@ -1081,12 +1138,13 @@ detail::enable_if_t::value, T> move(object &&obj) { if (obj.ref_count() > 1) { #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) throw cast_error( - "Unable to cast Python instance to C++ rvalue: instance has multiple references" - " (#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); + "Unable to cast Python " + str(type::handle_of(obj)).cast() + + " instance to C++ rvalue: instance has multiple references" + " (#define PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); #else - throw cast_error("Unable to move from Python " + (std::string) str(type::handle_of(obj)) - + " instance to C++ " + type_id() - + " instance: instance has multiple references"); + throw cast_error("Unable to move from Python " + + str(type::handle_of(obj)).cast() + " instance to C++ " + + type_id() + " instance: instance has multiple references"); #endif } @@ -1191,9 +1249,10 @@ PYBIND11_NAMESPACE_END(detail) // The overloads could coexist, i.e. the #if is not strictly speaking needed, // but it is an easy minor optimization. #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) -inline cast_error cast_error_unable_to_convert_call_arg() { - return cast_error("Unable to convert call argument to Python object (#define " - "PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); +inline cast_error cast_error_unable_to_convert_call_arg(const std::string &name) { + return cast_error("Unable to convert call argument '" + name + + "' to Python object (#define " + "PYBIND11_DETAILED_ERROR_MESSAGES or compile in debug mode for details)"); } #else inline cast_error cast_error_unable_to_convert_call_arg(const std::string &name, @@ -1216,7 +1275,7 @@ tuple make_tuple(Args &&...args_) { for (size_t i = 0; i < args.size(); i++) { if (!args[i]) { #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) - throw cast_error_unable_to_convert_call_arg(); + throw cast_error_unable_to_convert_call_arg(std::to_string(i)); #else std::array argtypes{{type_id()...}}; throw cast_error_unable_to_convert_call_arg(std::to_string(i), argtypes[i]); @@ -1336,7 +1395,15 @@ inline namespace literals { /** \rst String literal version of `arg` \endrst */ -constexpr arg operator"" _a(const char *name, size_t) { return arg(name); } +constexpr arg +#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ < 5 +operator"" _a // gcc 4.8.5 insists on having a space (hard error). +#else +operator""_a // clang 17 generates a deprecation warning if there is a space. +#endif + (const char *name, size_t) { + return arg(name); +} } // namespace literals PYBIND11_NAMESPACE_BEGIN(detail) @@ -1506,7 +1573,7 @@ class unpacking_collector { detail::make_caster::cast(std::forward(x), policy, {})); if (!o) { #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) - throw cast_error_unable_to_convert_call_arg(); + throw cast_error_unable_to_convert_call_arg(std::to_string(args_list.size())); #else throw cast_error_unable_to_convert_call_arg(std::to_string(args_list.size()), type_id()); @@ -1538,7 +1605,7 @@ class unpacking_collector { } if (!a.value) { #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) - throw cast_error_unable_to_convert_call_arg(); + throw cast_error_unable_to_convert_call_arg(a.name); #else throw cast_error_unable_to_convert_call_arg(a.name, a.type); #endif diff --git a/lib/pybind11/include/pybind11/detail/class.h b/lib/pybind11/include/pybind11/detail/class.h index 528e716f..0b9ea42d 100644 --- a/lib/pybind11/include/pybind11/detail/class.h +++ b/lib/pybind11/include/pybind11/detail/class.h @@ -86,17 +86,16 @@ inline PyTypeObject *make_static_property_type() { type->tp_descr_get = pybind11_static_get; type->tp_descr_set = pybind11_static_set; - if (PyType_Ready(type) < 0) { - pybind11_fail("make_static_property_type(): failure in PyType_Ready()!"); - } - # if PY_VERSION_HEX >= 0x030C0000 - // PRE 3.12 FEATURE FREEZE. PLEASE REVIEW AFTER FREEZE. // Since Python-3.12 property-derived types are required to // have dynamic attributes (to set `__doc__`) enable_dynamic_attributes(heap_type); # endif + if (PyType_Ready(type) < 0) { + pybind11_fail("make_static_property_type(): failure in PyType_Ready()!"); + } + setattr((PyObject *) type, "__module__", str("pybind11_builtins")); PYBIND11_SET_OLDPY_QUALNAME(type, name_obj); @@ -189,12 +188,10 @@ extern "C" inline PyObject *pybind11_meta_call(PyObject *type, PyObject *args, P return nullptr; } - // This must be a pybind11 instance - auto *instance = reinterpret_cast(self); - // Ensure that the base __init__ function(s) were called - for (const auto &vh : values_and_holders(instance)) { - if (!vh.holder_constructed()) { + values_and_holders vhs(self); + for (const auto &vh : vhs) { + if (!vh.holder_constructed() && !vhs.is_redundant_value_and_holder(vh)) { PyErr_Format(PyExc_TypeError, "%.200s.__init__() must be called when overriding __init__", get_fully_qualified_tp_name(vh.type->type).c_str()); @@ -375,7 +372,7 @@ extern "C" inline PyObject *pybind11_object_new(PyTypeObject *type, PyObject *, extern "C" inline int pybind11_object_init(PyObject *self, PyObject *, PyObject *) { PyTypeObject *type = Py_TYPE(self); std::string msg = get_fully_qualified_tp_name(type) + ": No constructor defined!"; - PyErr_SetString(PyExc_TypeError, msg.c_str()); + set_error(PyExc_TypeError, msg.c_str()); return -1; } @@ -445,9 +442,17 @@ inline void clear_instance(PyObject *self) { /// Instance destructor function for all pybind11 types. It calls `type_info.dealloc` /// to destroy the C++ object itself, while the rest is Python bookkeeping. extern "C" inline void pybind11_object_dealloc(PyObject *self) { + auto *type = Py_TYPE(self); + + // If this is a GC tracked object, untrack it first + // Note that the track call is implicitly done by the + // default tp_alloc, which we never override. + if (PyType_HasFeature(type, Py_TPFLAGS_HAVE_GC) != 0) { + PyObject_GC_UnTrack(self); + } + clear_instance(self); - auto *type = Py_TYPE(self); type->tp_free(self); #if PY_VERSION_HEX < 0x03080000 @@ -514,8 +519,12 @@ inline PyObject *make_object_base_type(PyTypeObject *metaclass) { /// dynamic_attr: Allow the garbage collector to traverse the internal instance `__dict__`. extern "C" inline int pybind11_traverse(PyObject *self, visitproc visit, void *arg) { +#if PY_VERSION_HEX >= 0x030D0000 + PyObject_VisitManagedDict(self, visit, arg); +#else PyObject *&dict = *_PyObject_GetDictPtr(self); Py_VISIT(dict); +#endif // https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_traverse #if PY_VERSION_HEX >= 0x03090000 Py_VISIT(Py_TYPE(self)); @@ -525,8 +534,12 @@ extern "C" inline int pybind11_traverse(PyObject *self, visitproc visit, void *a /// dynamic_attr: Allow the GC to clear the dictionary. extern "C" inline int pybind11_clear(PyObject *self) { +#if PY_VERSION_HEX >= 0x030D0000 + PyObject_ClearManagedDict(self); +#else PyObject *&dict = *_PyObject_GetDictPtr(self); Py_CLEAR(dict); +#endif return 0; } @@ -571,7 +584,7 @@ extern "C" inline int pybind11_getbuffer(PyObject *obj, Py_buffer *view, int fla if (view) { view->obj = nullptr; } - PyErr_SetString(PyExc_BufferError, "pybind11_getbuffer(): Internal error"); + set_error(PyExc_BufferError, "pybind11_getbuffer(): Internal error"); return -1; } std::memset(view, 0, sizeof(Py_buffer)); @@ -579,7 +592,7 @@ extern "C" inline int pybind11_getbuffer(PyObject *obj, Py_buffer *view, int fla if ((flags & PyBUF_WRITABLE) == PyBUF_WRITABLE && info->readonly) { delete info; // view->obj = nullptr; // Was just memset to 0, so not necessary - PyErr_SetString(PyExc_BufferError, "Writable buffer requested for readonly storage"); + set_error(PyExc_BufferError, "Writable buffer requested for readonly storage"); return -1; } view->obj = obj; diff --git a/lib/pybind11/include/pybind11/detail/common.h b/lib/pybind11/include/pybind11/detail/common.h index 9b74323f..83800e96 100644 --- a/lib/pybind11/include/pybind11/detail/common.h +++ b/lib/pybind11/include/pybind11/detail/common.h @@ -10,15 +10,76 @@ #pragma once #define PYBIND11_VERSION_MAJOR 2 -#define PYBIND11_VERSION_MINOR 11 +#define PYBIND11_VERSION_MINOR 12 #define PYBIND11_VERSION_PATCH 0.dev1 // Similar to Python's convention: https://docs.python.org/3/c-api/apiabiversion.html // Additional convention: 0xD = dev -#define PYBIND11_VERSION_HEX 0x020B00D1 +#define PYBIND11_VERSION_HEX 0x020C00D1 + +// Define some generic pybind11 helper macros for warning management. +// +// Note that compiler-specific push/pop pairs are baked into the +// PYBIND11_NAMESPACE_BEGIN/PYBIND11_NAMESPACE_END pair of macros. Therefore manual +// PYBIND11_WARNING_PUSH/PYBIND11_WARNING_POP are usually only needed in `#include` sections. +// +// If you find you need to suppress a warning, please try to make the suppression as local as +// possible using these macros. Please also be sure to push/pop with the pybind11 macros. Please +// only use compiler specifics if you need to check specific versions, e.g. Apple Clang vs. vanilla +// Clang. +#if defined(_MSC_VER) +# define PYBIND11_COMPILER_MSVC +# define PYBIND11_PRAGMA(...) __pragma(__VA_ARGS__) +# define PYBIND11_WARNING_PUSH PYBIND11_PRAGMA(warning(push)) +# define PYBIND11_WARNING_POP PYBIND11_PRAGMA(warning(pop)) +#elif defined(__INTEL_COMPILER) +# define PYBIND11_COMPILER_INTEL +# define PYBIND11_PRAGMA(...) _Pragma(#__VA_ARGS__) +# define PYBIND11_WARNING_PUSH PYBIND11_PRAGMA(warning push) +# define PYBIND11_WARNING_POP PYBIND11_PRAGMA(warning pop) +#elif defined(__clang__) +# define PYBIND11_COMPILER_CLANG +# define PYBIND11_PRAGMA(...) _Pragma(#__VA_ARGS__) +# define PYBIND11_WARNING_PUSH PYBIND11_PRAGMA(clang diagnostic push) +# define PYBIND11_WARNING_POP PYBIND11_PRAGMA(clang diagnostic push) +#elif defined(__GNUC__) +# define PYBIND11_COMPILER_GCC +# define PYBIND11_PRAGMA(...) _Pragma(#__VA_ARGS__) +# define PYBIND11_WARNING_PUSH PYBIND11_PRAGMA(GCC diagnostic push) +# define PYBIND11_WARNING_POP PYBIND11_PRAGMA(GCC diagnostic pop) +#endif + +#ifdef PYBIND11_COMPILER_MSVC +# define PYBIND11_WARNING_DISABLE_MSVC(name) PYBIND11_PRAGMA(warning(disable : name)) +#else +# define PYBIND11_WARNING_DISABLE_MSVC(name) +#endif + +#ifdef PYBIND11_COMPILER_CLANG +# define PYBIND11_WARNING_DISABLE_CLANG(name) PYBIND11_PRAGMA(clang diagnostic ignored name) +#else +# define PYBIND11_WARNING_DISABLE_CLANG(name) +#endif + +#ifdef PYBIND11_COMPILER_GCC +# define PYBIND11_WARNING_DISABLE_GCC(name) PYBIND11_PRAGMA(GCC diagnostic ignored name) +#else +# define PYBIND11_WARNING_DISABLE_GCC(name) +#endif + +#ifdef PYBIND11_COMPILER_INTEL +# define PYBIND11_WARNING_DISABLE_INTEL(name) PYBIND11_PRAGMA(warning disable name) +#else +# define PYBIND11_WARNING_DISABLE_INTEL(name) +#endif + +#define PYBIND11_NAMESPACE_BEGIN(name) \ + namespace name { \ + PYBIND11_WARNING_PUSH -#define PYBIND11_NAMESPACE_BEGIN(name) namespace name { -#define PYBIND11_NAMESPACE_END(name) } +#define PYBIND11_NAMESPACE_END(name) \ + PYBIND11_WARNING_POP \ + } // Robust support for some features and loading modules compiled against different pybind versions // requires forcing hidden visibility on pybind code, so we enforce this by setting the attribute @@ -57,6 +118,14 @@ # endif #endif +#if defined(PYBIND11_CPP20) +# define PYBIND11_CONSTINIT constinit +# define PYBIND11_DTOR_CONSTEXPR constexpr +#else +# define PYBIND11_CONSTINIT +# define PYBIND11_DTOR_CONSTEXPR +#endif + // Compiler version assertions #if defined(__INTEL_COMPILER) # if __INTEL_COMPILER < 1800 @@ -151,9 +220,9 @@ /// Include Python header, disable linking to pythonX_d.lib on Windows in debug mode #if defined(_MSC_VER) -# pragma warning(push) +PYBIND11_WARNING_PUSH +PYBIND11_WARNING_DISABLE_MSVC(4505) // C4505: 'PySlice_GetIndicesEx': unreferenced local function has been removed (PyPy only) -# pragma warning(disable : 4505) # if defined(_DEBUG) && !defined(Py_DEBUG) // Workaround for a VS 2022 issue. // NOTE: This workaround knowingly violates the Python.h include order requirement: @@ -236,7 +305,7 @@ # define _DEBUG # undef PYBIND11_DEBUG_MARKER # endif -# pragma warning(pop) +PYBIND11_WARNING_POP #endif #include @@ -262,6 +331,12 @@ # define PYBIND11_HAS_U8STRING #endif +// See description of PR #4246: +#if !defined(PYBIND11_NO_ASSERT_GIL_HELD_INCREF_DECREF) && !defined(NDEBUG) \ + && !defined(PYPY_VERSION) && !defined(PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF) +# define PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF +#endif + // #define PYBIND11_STR_LEGACY_PERMISSIVE // If DEFINED, pybind11::str can hold PyUnicodeObject or PyBytesObject // (probably surprising and never documented, but this was the @@ -332,7 +407,7 @@ return nullptr; \ } \ catch (const std::exception &e) { \ - PyErr_SetString(PyExc_ImportError, e.what()); \ + ::pybind11::set_error(PyExc_ImportError, e.what()); \ return nullptr; \ } @@ -366,7 +441,7 @@ /** \rst This macro creates the entry point that will be invoked when the Python interpreter - imports an extension module. The module name is given as the fist argument and it + imports an extension module. The module name is given as the first argument and it should not be in quotes. The second macro argument defines a variable of type `py::module_` which can be used to initialize the module. @@ -591,6 +666,10 @@ template using remove_cvref_t = typename remove_cvref::type; #endif +/// Example usage: is_same_ignoring_cvref::value +template +using is_same_ignoring_cvref = std::is_same, U>; + /// Index sequences #if defined(PYBIND11_CPP14) using std::index_sequence; @@ -684,7 +763,16 @@ template struct remove_class { using type = R(A...); }; - +#ifdef __cpp_noexcept_function_type +template +struct remove_class { + using type = R(A...); +}; +template +struct remove_class { + using type = R(A...); +}; +#endif /// Helper template to strip away type modifiers template struct intrinsic_type { @@ -942,6 +1030,15 @@ PYBIND11_RUNTIME_EXCEPTION(reference_cast_error, PyExc_RuntimeError) /// Used in template struct format_descriptor {}; +template +struct format_descriptor< + T, + detail::enable_if_t::value>> { + static constexpr const char c = 'O'; + static constexpr const char value[2] = {c, '\0'}; + static std::string format() { return std::string(1, c); } +}; + PYBIND11_NAMESPACE_BEGIN(detail) // Returns the index of the given type in the type char array below, and in the list in numpy.h // The order here is: bool; 8 ints ((signed,unsigned)x(8,16,32,64)bits); float,double,long double; @@ -1136,17 +1233,6 @@ constexpr # define PYBIND11_WORKAROUND_INCORRECT_GCC_UNUSED_BUT_SET_PARAMETER(...) #endif -#if defined(_MSC_VER) // All versions (as of July 2021). - -// warning C4127: Conditional expression is constant -constexpr inline bool silence_msvc_c4127(bool cond) { return cond; } - -# define PYBIND11_SILENCE_MSVC_C4127(...) ::pybind11::detail::silence_msvc_c4127(__VA_ARGS__) - -#else -# define PYBIND11_SILENCE_MSVC_C4127(...) __VA_ARGS__ -#endif - #if defined(__clang__) \ && (defined(__apple_build_version__) /* AppleClang 13.0.0.13000029 was the only data point \ available. */ \ @@ -1166,8 +1252,9 @@ constexpr inline bool silence_msvc_c4127(bool cond) { return cond; } #endif // Pybind offers detailed error messages by default for all builts that are debug (through the -// negation of ndebug). This can also be manually enabled by users, for any builds, through -// defining PYBIND11_DETAILED_ERROR_MESSAGES. +// negation of NDEBUG). This can also be manually enabled by users, for any builds, through +// defining PYBIND11_DETAILED_ERROR_MESSAGES. This information is primarily useful for those +// who are writing (as opposed to merely using) libraries that use pybind11. #if !defined(PYBIND11_DETAILED_ERROR_MESSAGES) && !defined(NDEBUG) # define PYBIND11_DETAILED_ERROR_MESSAGES #endif diff --git a/lib/pybind11/include/pybind11/detail/descr.h b/lib/pybind11/include/pybind11/detail/descr.h index e7a5e2c1..635614b0 100644 --- a/lib/pybind11/include/pybind11/detail/descr.h +++ b/lib/pybind11/include/pybind11/detail/descr.h @@ -143,11 +143,24 @@ constexpr descr concat(const descr &descr) { return descr; } +#ifdef __cpp_fold_expressions +template +constexpr descr operator,(const descr &a, + const descr &b) { + return a + const_name(", ") + b; +} + +template +constexpr auto concat(const descr &d, const Args &...args) { + return (d, ..., args); +} +#else template constexpr auto concat(const descr &d, const Args &...args) -> decltype(std::declval>() + concat(args...)) { return d + const_name(", ") + concat(args...); } +#endif template constexpr descr type_descr(const descr &descr) { diff --git a/lib/pybind11/include/pybind11/detail/init.h b/lib/pybind11/include/pybind11/detail/init.h index 05f4fe54..4509bd13 100644 --- a/lib/pybind11/include/pybind11/detail/init.h +++ b/lib/pybind11/include/pybind11/detail/init.h @@ -12,6 +12,9 @@ #include "class.h" PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +PYBIND11_WARNING_DISABLE_MSVC(4127) + PYBIND11_NAMESPACE_BEGIN(detail) template <> @@ -62,7 +65,7 @@ constexpr bool is_alias(void *) { } // Constructs and returns a new object; if the given arguments don't map to a constructor, we fall -// back to brace aggregate initiailization so that for aggregate initialization can be used with +// back to brace aggregate initialization so that for aggregate initialization can be used with // py::init, e.g. `py::init` to initialize a `struct T { int a; int b; }`. For // non-aggregate types, we need to use an ordinary T(...) constructor (invoking as `T{...}` usually // works, but will not do the expected thing when `T` has an `initializer_list` constructor). @@ -115,7 +118,7 @@ template void construct(value_and_holder &v_h, Cpp *ptr, bool need_alias) { PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(need_alias); no_nullptr(ptr); - if (PYBIND11_SILENCE_MSVC_C4127(Class::has_alias) && need_alias && !is_alias(ptr)) { + if (Class::has_alias && need_alias && !is_alias(ptr)) { // We're going to try to construct an alias by moving the cpp type. Whether or not // that succeeds, we still need to destroy the original cpp pointer (either the // moved away leftover, if the alias construction works, or the value itself if we @@ -156,7 +159,7 @@ void construct(value_and_holder &v_h, Holder holder, bool need_alias) { auto *ptr = holder_helper>::get(holder); no_nullptr(ptr); // If we need an alias, check that the held pointer is actually an alias instance - if (PYBIND11_SILENCE_MSVC_C4127(Class::has_alias) && need_alias && !is_alias(ptr)) { + if (Class::has_alias && need_alias && !is_alias(ptr)) { throw type_error("pybind11::init(): construction failed: returned holder-wrapped instance " "is not an alias instance"); } @@ -172,9 +175,9 @@ void construct(value_and_holder &v_h, Holder holder, bool need_alias) { template void construct(value_and_holder &v_h, Cpp &&result, bool need_alias) { PYBIND11_WORKAROUND_INCORRECT_MSVC_C4100(need_alias); - static_assert(std::is_move_constructible>::value, + static_assert(is_move_constructible>::value, "pybind11::init() return-by-value factory function requires a movable class"); - if (PYBIND11_SILENCE_MSVC_C4127(Class::has_alias) && need_alias) { + if (Class::has_alias && need_alias) { construct_alias_from_cpp(is_alias_constructible{}, v_h, std::move(result)); } else { v_h.value_ptr() = new Cpp(std::move(result)); @@ -187,7 +190,7 @@ void construct(value_and_holder &v_h, Cpp &&result, bool need_alias) { template void construct(value_and_holder &v_h, Alias &&result, bool) { static_assert( - std::is_move_constructible>::value, + is_move_constructible>::value, "pybind11::init() return-by-alias-value factory function requires a movable alias class"); v_h.value_ptr() = new Alias(std::move(result)); } @@ -206,10 +209,11 @@ struct constructor { extra...); } - template , Args...>::value, - int> = 0> + template < + typename Class, + typename... Extra, + enable_if_t, Args...>::value, int> + = 0> static void execute(Class &cl, const Extra &...extra) { cl.def( "__init__", @@ -226,10 +230,11 @@ struct constructor { extra...); } - template , Args...>::value, - int> = 0> + template < + typename Class, + typename... Extra, + enable_if_t, Args...>::value, int> + = 0> static void execute(Class &cl, const Extra &...extra) { cl.def( "__init__", @@ -245,10 +250,11 @@ struct constructor { // Implementing class for py::init_alias<...>() template struct alias_constructor { - template , Args...>::value, - int> = 0> + template < + typename Class, + typename... Extra, + enable_if_t, Args...>::value, int> + = 0> static void execute(Class &cl, const Extra &...extra) { cl.def( "__init__", diff --git a/lib/pybind11/include/pybind11/detail/internals.h b/lib/pybind11/include/pybind11/detail/internals.h index 6fd61098..ab399016 100644 --- a/lib/pybind11/include/pybind11/detail/internals.h +++ b/lib/pybind11/include/pybind11/detail/internals.h @@ -34,9 +34,19 @@ /// further ABI-incompatible changes may be made before the ABI is officially /// changed to the new version. #ifndef PYBIND11_INTERNALS_VERSION -# define PYBIND11_INTERNALS_VERSION 4 +# if PY_VERSION_HEX >= 0x030C0000 || defined(_MSC_VER) +// Version bump for Python 3.12+, before first 3.12 beta release. +// Version bump for MSVC piggy-backed on PR #4779. See comments there. +# define PYBIND11_INTERNALS_VERSION 5 +# else +# define PYBIND11_INTERNALS_VERSION 4 +# endif #endif +// This requirement is mainly to reduce the support burden (see PR #4570). +static_assert(PY_VERSION_HEX < 0x030C0000 || PYBIND11_INTERNALS_VERSION >= 5, + "pybind11 ABI version 5 is the minimum for Python 3.12+"); + PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) using ExceptionTranslator = void (*)(std::exception_ptr); @@ -114,7 +124,8 @@ inline void tls_replace_value(PYBIND11_TLS_KEY_REF key, void *value) { // libstdc++, this doesn't happen: equality and the type_index hash are based on the type name, // which works. If not under a known-good stl, provide our own name-based hash and equality // functions that use the type name. -#if defined(__GLIBCXX__) +#if (PYBIND11_INTERNALS_VERSION <= 4 && defined(__GLIBCXX__)) \ + || (PYBIND11_INTERNALS_VERSION >= 5 && !defined(_LIBCPP_VERSION)) inline bool same_type(const std::type_info &lhs, const std::type_info &rhs) { return lhs == rhs; } using type_hash = std::hash; using type_equal_to = std::equal_to; @@ -281,9 +292,12 @@ struct type_info { #endif /// On Linux/OSX, changes in __GXX_ABI_VERSION__ indicate ABI incompatibility. +/// On MSVC, changes in _MSC_VER may indicate ABI incompatibility (#2898). #ifndef PYBIND11_BUILD_ABI # if defined(__GXX_ABI_VERSION) # define PYBIND11_BUILD_ABI "_cxxabi" PYBIND11_TOSTRING(__GXX_ABI_VERSION) +# elif defined(_MSC_VER) +# define PYBIND11_BUILD_ABI "_mscver" PYBIND11_TOSTRING(_MSC_VER) # else # define PYBIND11_BUILD_ABI "" # endif @@ -342,7 +356,7 @@ inline bool raise_err(PyObject *exc_type, const char *msg) { raise_from(exc_type, msg); return true; } - PyErr_SetString(exc_type, msg); + set_error(exc_type, msg); return false; } @@ -421,6 +435,40 @@ inline void translate_local_exception(std::exception_ptr p) { } #endif +inline object get_python_state_dict() { + object state_dict; +#if PYBIND11_INTERNALS_VERSION <= 4 || PY_VERSION_HEX < 0x03080000 || defined(PYPY_VERSION) + state_dict = reinterpret_borrow(PyEval_GetBuiltins()); +#else +# if PY_VERSION_HEX < 0x03090000 + PyInterpreterState *istate = _PyInterpreterState_Get(); +# else + PyInterpreterState *istate = PyInterpreterState_Get(); +# endif + if (istate) { + state_dict = reinterpret_borrow(PyInterpreterState_GetDict(istate)); + } +#endif + if (!state_dict) { + raise_from(PyExc_SystemError, "pybind11::detail::get_python_state_dict() FAILED"); + throw error_already_set(); + } + return state_dict; +} + +inline object get_internals_obj_from_state_dict(handle state_dict) { + return reinterpret_borrow(dict_getitemstring(state_dict.ptr(), PYBIND11_INTERNALS_ID)); +} + +inline internals **get_internals_pp_from_capsule(handle obj) { + void *raw_ptr = PyCapsule_GetPointer(obj.ptr(), /*name=*/nullptr); + if (raw_ptr == nullptr) { + raise_from(PyExc_SystemError, "pybind11::detail::get_internals_pp_from_capsule() FAILED"); + throw error_already_set(); + } + return static_cast(raw_ptr); +} + /// Return a reference to the current `internals` data PYBIND11_NOINLINE internals &get_internals() { auto **&internals_pp = get_internals_pp(); @@ -445,12 +493,12 @@ PYBIND11_NOINLINE internals &get_internals() { #endif error_scope err_scope; - PYBIND11_STR_TYPE id(PYBIND11_INTERNALS_ID); - auto builtins = handle(PyEval_GetBuiltins()); - if (builtins.contains(id) && isinstance(builtins[id])) { - internals_pp = static_cast(capsule(builtins[id])); - - // We loaded builtins through python's builtins, which means that our `error_already_set` + dict state_dict = get_python_state_dict(); + if (object internals_obj = get_internals_obj_from_state_dict(state_dict)) { + internals_pp = get_internals_pp_from_capsule(internals_obj); + } + if (internals_pp && *internals_pp) { + // We loaded the internals through `state_dict`, which means that our `error_already_set` // and `builtin_exception` may be different local classes than the ones set up in the // initial exception translator, below, so add another for our local exception classes. // @@ -468,16 +516,15 @@ PYBIND11_NOINLINE internals &get_internals() { internals_ptr = new internals(); #if defined(WITH_THREAD) -# if PY_VERSION_HEX < 0x03090000 - PyEval_InitThreads(); -# endif PyThreadState *tstate = PyThreadState_Get(); + // NOLINTNEXTLINE(bugprone-assignment-in-if-condition) if (!PYBIND11_TLS_KEY_CREATE(internals_ptr->tstate)) { pybind11_fail("get_internals: could not successfully initialize the tstate TSS key!"); } PYBIND11_TLS_REPLACE_VALUE(internals_ptr->tstate, tstate); # if PYBIND11_INTERNALS_VERSION > 4 + // NOLINTNEXTLINE(bugprone-assignment-in-if-condition) if (!PYBIND11_TLS_KEY_CREATE(internals_ptr->loader_life_support_tls_key)) { pybind11_fail("get_internals: could not successfully initialize the " "loader_life_support TSS key!"); @@ -485,7 +532,7 @@ PYBIND11_NOINLINE internals &get_internals() { # endif internals_ptr->istate = tstate->interp; #endif - builtins[id] = capsule(internals_pp); + state_dict[PYBIND11_INTERNALS_ID] = capsule(internals_pp); internals_ptr->registered_exception_translators.push_front(&translate_exception); internals_ptr->static_property_type = make_static_property_type(); internals_ptr->default_metaclass = make_default_metaclass(); @@ -517,6 +564,7 @@ struct local_internals { struct shared_loader_life_support_data { PYBIND11_TLS_KEY_INIT(loader_life_support_tls_key) shared_loader_life_support_data() { + // NOLINTNEXTLINE(bugprone-assignment-in-if-condition) if (!PYBIND11_TLS_KEY_CREATE(loader_life_support_tls_key)) { pybind11_fail("local_internals: could not successfully initialize the " "loader_life_support TLS key!"); diff --git a/lib/pybind11/include/pybind11/detail/type_caster_base.h b/lib/pybind11/include/pybind11/detail/type_caster_base.h index 21f69c28..476646ee 100644 --- a/lib/pybind11/include/pybind11/detail/type_caster_base.h +++ b/lib/pybind11/include/pybind11/detail/type_caster_base.h @@ -102,8 +102,22 @@ class loader_life_support { inline std::pair all_type_info_get_cache(PyTypeObject *type); +// Band-aid workaround to fix a subtle but serious bug in a minimalistic fashion. See PR #4762. +inline void all_type_info_add_base_most_derived_first(std::vector &bases, + type_info *addl_base) { + for (auto it = bases.begin(); it != bases.end(); it++) { + type_info *existing_base = *it; + if (PyType_IsSubtype(addl_base->type, existing_base->type) != 0) { + bases.insert(it, addl_base); + return; + } + } + bases.push_back(addl_base); +} + // Populates a just-created cache entry. PYBIND11_NOINLINE void all_type_info_populate(PyTypeObject *t, std::vector &bases) { + assert(bases.empty()); std::vector check; for (handle parent : reinterpret_borrow(t->tp_bases)) { check.push_back((PyTypeObject *) parent.ptr()); @@ -136,7 +150,7 @@ PYBIND11_NOINLINE void all_type_info_populate(PyTypeObject *t, std::vectortp_bases) { @@ -258,9 +272,9 @@ struct value_and_holder { // Main constructor for a found value/holder: value_and_holder(instance *i, const detail::type_info *type, size_t vpos, size_t index) - : inst{i}, index{index}, type{type}, vh{inst->simple_layout - ? inst->simple_value_holder - : &inst->nonsimple.values_and_holders[vpos]} {} + : inst{i}, index{index}, type{type}, + vh{inst->simple_layout ? inst->simple_value_holder + : &inst->nonsimple.values_and_holders[vpos]} {} // Default constructor (used to signal a value-and-holder not found by get_value_and_holder()) value_and_holder() = default; @@ -322,18 +336,29 @@ struct values_and_holders { explicit values_and_holders(instance *inst) : inst{inst}, tinfo(all_type_info(Py_TYPE(inst))) {} + explicit values_and_holders(PyObject *obj) + : inst{nullptr}, tinfo(all_type_info(Py_TYPE(obj))) { + if (!tinfo.empty()) { + inst = reinterpret_cast(obj); + } + } + struct iterator { private: instance *inst = nullptr; const type_vec *types = nullptr; value_and_holder curr; friend struct values_and_holders; - iterator(instance *inst, const type_vec *tinfo) - : inst{inst}, types{tinfo}, - curr(inst /* instance */, - types->empty() ? nullptr : (*types)[0] /* type info */, - 0, /* vpos: (non-simple types only): the first vptr comes first */ - 0 /* index */) {} + iterator(instance *inst, const type_vec *tinfo) : inst{inst}, types{tinfo} { + if (inst != nullptr) { + assert(!types->empty()); + curr = value_and_holder( + inst /* instance */, + (*types)[0] /* type info */, + 0, /* vpos: (non-simple types only): the first vptr comes first */ + 0 /* index */); + } + } // Past-the-end iterator: explicit iterator(size_t end) : curr(end) {} @@ -364,6 +389,16 @@ struct values_and_holders { } size_t size() { return tinfo.size(); } + + // Band-aid workaround to fix a subtle but serious bug in a minimalistic fashion. See PR #4762. + bool is_redundant_value_and_holder(const value_and_holder &vh) { + for (size_t i = 0; i < vh.index; i++) { + if (PyType_IsSubtype(tinfo[i]->type, tinfo[vh.index]->type) != 0) { + return true; + } + } + return false; + } }; /** @@ -486,8 +521,10 @@ PYBIND11_NOINLINE handle get_object_handle(const void *ptr, const detail::type_i inline PyThreadState *get_thread_state_unchecked() { #if defined(PYPY_VERSION) return PyThreadState_GET(); -#else +#elif PY_VERSION_HEX < 0x030D0000 return _PyThreadState_UncheckedGet(); +#else + return PyThreadState_GetUnchecked(); #endif } @@ -786,7 +823,7 @@ class type_caster_generic { std::string tname = rtti_type ? rtti_type->name() : cast_type.name(); detail::clean_type_id(tname); std::string msg = "Unregistered type : " + tname; - PyErr_SetString(PyExc_TypeError, msg.c_str()); + set_error(PyExc_TypeError, msg.c_str()); return {nullptr, nullptr}; } @@ -822,23 +859,179 @@ using movable_cast_op_type typename std::add_rvalue_reference>::type, typename std::add_lvalue_reference>::type>>; -// std::is_copy_constructible isn't quite enough: it lets std::vector (and similar) through when -// T is non-copyable, but code containing such a copy constructor fails to actually compile. -template -struct is_copy_constructible : std::is_copy_constructible {}; +// Does the container have a mapped type and is it recursive? +// Implemented by specializations below. +template +struct container_mapped_type_traits { + static constexpr bool has_mapped_type = false; + static constexpr bool has_recursive_mapped_type = false; +}; -// Specialization for types that appear to be copy constructible but also look like stl containers -// (we specifically check for: has `value_type` and `reference` with `reference = value_type&`): if -// so, copy constructability depends on whether the value_type is copy constructible. template -struct is_copy_constructible< +struct container_mapped_type_traits< Container, - enable_if_t< - all_of, - std::is_same, - // Avoid infinite recursion - negation>>::value>> - : is_copy_constructible {}; + typename std::enable_if< + std::is_same::value>::type> { + static constexpr bool has_mapped_type = true; + static constexpr bool has_recursive_mapped_type = true; +}; + +template +struct container_mapped_type_traits< + Container, + typename std::enable_if< + negation>::value>::type> { + static constexpr bool has_mapped_type = true; + static constexpr bool has_recursive_mapped_type = false; +}; + +// Does the container have a value type and is it recursive? +// Implemented by specializations below. +template +struct container_value_type_traits : std::false_type { + static constexpr bool has_value_type = false; + static constexpr bool has_recursive_value_type = false; +}; + +template +struct container_value_type_traits< + Container, + typename std::enable_if< + std::is_same::value>::type> { + static constexpr bool has_value_type = true; + static constexpr bool has_recursive_value_type = true; +}; + +template +struct container_value_type_traits< + Container, + typename std::enable_if< + negation>::value>::type> { + static constexpr bool has_value_type = true; + static constexpr bool has_recursive_value_type = false; +}; + +/* + * Tag to be used for representing the bottom of recursively defined types. + * Define this tag so we don't have to use void. + */ +struct recursive_bottom {}; + +/* + * Implementation detail of `recursive_container_traits` below. + * `T` is the `value_type` of the container, which might need to be modified to + * avoid recursive types and const types. + */ +template +struct impl_type_to_check_recursively { + /* + * If the container is recursive, then no further recursion should be done. + */ + using if_recursive = recursive_bottom; + /* + * Otherwise yield `T` unchanged. + */ + using if_not_recursive = T; +}; + +/* + * For pairs - only as value type of a map -, the first type should remove the `const`. + * Also, if the map is recursive, then the recursive checking should consider + * the first type only. + */ +template +struct impl_type_to_check_recursively, /* is_this_a_map = */ true> { + using if_recursive = typename std::remove_const::type; + using if_not_recursive = std::pair::type, B>; +}; + +/* + * Implementation of `recursive_container_traits` below. + */ +template +struct impl_recursive_container_traits { + using type_to_check_recursively = recursive_bottom; +}; + +template +struct impl_recursive_container_traits< + Container, + typename std::enable_if::has_value_type>::type> { + static constexpr bool is_recursive + = container_mapped_type_traits::has_recursive_mapped_type + || container_value_type_traits::has_recursive_value_type; + /* + * This member dictates which type Pybind11 should check recursively in traits + * such as `is_move_constructible`, `is_copy_constructible`, `is_move_assignable`, ... + * Direct access to `value_type` should be avoided: + * 1. `value_type` might recursively contain the type again + * 2. `value_type` of STL map types is `std::pair`, the `const` + * should be removed. + * + */ + using type_to_check_recursively = typename std::conditional< + is_recursive, + typename impl_type_to_check_recursively< + typename Container::value_type, + container_mapped_type_traits::has_mapped_type>::if_recursive, + typename impl_type_to_check_recursively< + typename Container::value_type, + container_mapped_type_traits::has_mapped_type>::if_not_recursive>::type; +}; + +/* + * This trait defines the `type_to_check_recursively` which is needed to properly + * handle recursively defined traits such as `is_move_constructible` without going + * into an infinite recursion. + * Should be used instead of directly accessing the `value_type`. + * It cancels the recursion by returning the `recursive_bottom` tag. + * + * The default definition of `type_to_check_recursively` is as follows: + * + * 1. By default, it is `recursive_bottom`, so that the recursion is canceled. + * 2. If the type is non-recursive and defines a `value_type`, then the `value_type` is used. + * If the `value_type` is a pair and a `mapped_type` is defined, + * then the `const` is removed from the first type. + * 3. If the type is recursive and `value_type` is not a pair, then `recursive_bottom` is returned. + * 4. If the type is recursive and `value_type` is a pair and a `mapped_type` is defined, + * then `const` is removed from the first type and the first type is returned. + * + * This behavior can be extended by the user as seen in test_stl_binders.cpp. + * + * This struct is exactly the same as impl_recursive_container_traits. + * The duplication achieves that user-defined specializations don't compete + * with internal specializations, but take precedence. + */ +template +struct recursive_container_traits : impl_recursive_container_traits {}; + +template +struct is_move_constructible + : all_of, + is_move_constructible< + typename recursive_container_traits::type_to_check_recursively>> {}; + +template <> +struct is_move_constructible : std::true_type {}; + +// Likewise for std::pair +// (after C++17 it is mandatory that the move constructor not exist when the two types aren't +// themselves move constructible, but this can not be relied upon when T1 or T2 are themselves +// containers). +template +struct is_move_constructible> + : all_of, is_move_constructible> {}; + +// std::is_copy_constructible isn't quite enough: it lets std::vector (and similar) through when +// T is non-copyable, but code containing such a copy constructor fails to actually compile. +template +struct is_copy_constructible + : all_of, + is_copy_constructible< + typename recursive_container_traits::type_to_check_recursively>> {}; + +template <> +struct is_copy_constructible : std::true_type {}; // Likewise for std::pair // (after C++17 it is mandatory that the copy constructor not exist when the two types aren't @@ -849,14 +1042,16 @@ struct is_copy_constructible> : all_of, is_copy_constructible> {}; // The same problems arise with std::is_copy_assignable, so we use the same workaround. -template -struct is_copy_assignable : std::is_copy_assignable {}; -template -struct is_copy_assignable, - std::is_same>::value>> - : is_copy_assignable {}; +template +struct is_copy_assignable + : all_of< + std::is_copy_assignable, + is_copy_assignable::type_to_check_recursively>> { +}; + +template <> +struct is_copy_assignable : std::true_type {}; + template struct is_copy_assignable> : all_of, is_copy_assignable> {}; @@ -994,7 +1189,7 @@ class type_caster_base : public type_caster_generic { return [](const void *arg) -> void * { return new T(*reinterpret_cast(arg)); }; } - template ::value>> + template ::value>> static auto make_move_constructor(const T *) -> decltype(new T(std::declval()), Constructor{}) { return [](const void *arg) -> void * { @@ -1006,5 +1201,14 @@ class type_caster_base : public type_caster_generic { static Constructor make_move_constructor(...) { return nullptr; } }; +PYBIND11_NOINLINE std::string type_info_description(const std::type_info &ti) { + if (auto *type_data = get_type_info(ti)) { + handle th((PyObject *) type_data->type); + return th.attr("__module__").cast() + '.' + + th.attr("__qualname__").cast(); + } + return clean_type_id(ti.name()); +} + PYBIND11_NAMESPACE_END(detail) PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/lib/pybind11/include/pybind11/eigen/common.h b/lib/pybind11/include/pybind11/eigen/common.h new file mode 100644 index 00000000..24f56d15 --- /dev/null +++ b/lib/pybind11/include/pybind11/eigen/common.h @@ -0,0 +1,9 @@ +// Copyright (c) 2023 The pybind Community. + +#pragma once + +// Common message for `static_assert()`s, which are useful to easily +// preempt much less obvious errors. +#define PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED \ + "Pointer types (in particular `PyObject *`) are not supported as scalar types for Eigen " \ + "types." diff --git a/lib/pybind11/include/pybind11/eigen/matrix.h b/lib/pybind11/include/pybind11/eigen/matrix.h index c30dac24..8d4342f8 100644 --- a/lib/pybind11/include/pybind11/eigen/matrix.h +++ b/lib/pybind11/include/pybind11/eigen/matrix.h @@ -10,35 +10,24 @@ #pragma once #include "../numpy.h" +#include "common.h" /* HINT: To suppress warnings originating from the Eigen headers, use -isystem. See also: https://stackoverflow.com/questions/2579576/i-dir-vs-isystem-dir https://stackoverflow.com/questions/1741816/isystem-for-ms-visual-studio-c-compiler */ -// The C4127 suppression was introduced for Eigen 3.4.0. In theory we could -// make it version specific, or even remove it later, but considering that -// 1. C4127 is generally far more distracting than useful for modern template code, and -// 2. we definitely want to ignore any MSVC warnings originating from Eigen code, -// it is probably best to keep this around indefinitely. -#if defined(_MSC_VER) -# pragma warning(push) -# pragma warning(disable : 4127) // C4127: conditional expression is constant -# pragma warning(disable : 5054) // https://github.com/pybind/pybind11/pull/3741 +PYBIND11_WARNING_PUSH +PYBIND11_WARNING_DISABLE_MSVC(5054) // https://github.com/pybind/pybind11/pull/3741 // C5054: operator '&': deprecated between enumerations of different types -#elif defined(__MINGW32__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#if defined(__MINGW32__) +PYBIND11_WARNING_DISABLE_GCC("-Wmaybe-uninitialized") #endif #include #include -#if defined(_MSC_VER) -# pragma warning(pop) -#elif defined(__MINGW32__) -# pragma GCC diagnostic pop -#endif +PYBIND11_WARNING_POP // Eigen prior to 3.2.7 doesn't have proper move constructors--but worse, some classes get implicit // move constructors that break things. We could detect this an explicitly copy, but an extra copy @@ -48,6 +37,8 @@ static_assert(EIGEN_VERSION_AT_LEAST(3, 2, 7), PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_WARNING_DISABLE_MSVC(4127) + // Provide a convenience alias for easier pass-by-ref usage with fully dynamic strides: using EigenDStride = Eigen::Stride; template @@ -189,8 +180,7 @@ struct EigenProps { EigenIndex np_rows = a.shape(0), np_cols = a.shape(1), np_rstride = a.strides(0) / static_cast(sizeof(Scalar)), np_cstride = a.strides(1) / static_cast(sizeof(Scalar)); - if ((PYBIND11_SILENCE_MSVC_C4127(fixed_rows) && np_rows != rows) - || (PYBIND11_SILENCE_MSVC_C4127(fixed_cols) && np_cols != cols)) { + if ((fixed_rows && np_rows != rows) || (fixed_cols && np_cols != cols)) { return false; } @@ -203,7 +193,7 @@ struct EigenProps { stride = a.strides(0) / static_cast(sizeof(Scalar)); if (vector) { // Eigen type is a compile-time vector - if (PYBIND11_SILENCE_MSVC_C4127(fixed) && size != n) { + if (fixed && size != n) { return false; // Vector size mismatch } return {rows == 1 ? 1 : n, cols == 1 ? 1 : n, stride}; @@ -220,7 +210,7 @@ struct EigenProps { } return {1, n, stride}; } // Otherwise it's either fully dynamic, or column dynamic; both become a column vector - if (PYBIND11_SILENCE_MSVC_C4127(fixed_rows) && rows != n) { + if (fixed_rows && rows != n) { return false; } return {n, 1, stride}; @@ -298,6 +288,8 @@ handle eigen_encapsulate(Type *src) { template struct type_caster::value>> { using Scalar = typename Type::Scalar; + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); using props = EigenProps; bool load(handle src, bool convert) { @@ -416,6 +408,9 @@ struct type_caster::value>> { // Base class for casting reference/map/block/etc. objects back to python. template struct eigen_map_caster { + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); + private: using props = EigenProps; @@ -468,6 +463,8 @@ struct type_caster< using Type = Eigen::Ref; using props = EigenProps; using Scalar = typename props::Scalar; + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); using MapType = Eigen::Map; using Array = array_t struct type_caster::value>> { + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); + protected: using Matrix = Eigen::Matrix; @@ -643,6 +643,8 @@ struct type_caster::value>> { template struct type_caster::value>> { using Scalar = typename Type::Scalar; + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); using StorageIndex = remove_reference_t().outerIndexPtr())>; using Index = typename Type::Index; static constexpr bool rowMajor = Type::IsRowMajor; diff --git a/lib/pybind11/include/pybind11/eigen/tensor.h b/lib/pybind11/include/pybind11/eigen/tensor.h index a823c0f3..25d12bac 100644 --- a/lib/pybind11/include/pybind11/eigen/tensor.h +++ b/lib/pybind11/include/pybind11/eigen/tensor.h @@ -8,33 +8,31 @@ #pragma once #include "../numpy.h" +#include "common.h" #if defined(__GNUC__) && !defined(__clang__) && !defined(__INTEL_COMPILER) static_assert(__GNUC__ > 5, "Eigen Tensor support in pybind11 requires GCC > 5.0"); #endif -#if defined(_MSC_VER) -# pragma warning(push) -# pragma warning(disable : 4554) // Tensor.h warning -# pragma warning(disable : 4127) // Tensor.h warning -#elif defined(__MINGW32__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +// Disable warnings for Eigen +PYBIND11_WARNING_PUSH +PYBIND11_WARNING_DISABLE_MSVC(4554) +PYBIND11_WARNING_DISABLE_MSVC(4127) +#if defined(__MINGW32__) +PYBIND11_WARNING_DISABLE_GCC("-Wmaybe-uninitialized") #endif #include -#if defined(_MSC_VER) -# pragma warning(pop) -#elif defined(__MINGW32__) -# pragma GCC diagnostic pop -#endif +PYBIND11_WARNING_POP static_assert(EIGEN_VERSION_AT_LEAST(3, 3, 0), "Eigen Tensor support in pybind11 requires Eigen >= 3.3.0"); PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_WARNING_DISABLE_MSVC(4127) + PYBIND11_NAMESPACE_BEGIN(detail) inline bool is_tensor_aligned(const void *data) { @@ -138,10 +136,8 @@ struct get_tensor_descriptor { // // We need to disable the type-limits warning for the inner loop when size = 0. -#if defined(__GNUC__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wtype-limits" -#endif +PYBIND11_WARNING_PUSH +PYBIND11_WARNING_DISABLE_GCC("-Wtype-limits") template std::vector convert_dsizes_to_vector(const Eigen::DSizes &arr) { @@ -165,12 +161,12 @@ Eigen::DSizes get_shape_for_array(const array &arr) { return result; } -#if defined(__GNUC__) -# pragma GCC diagnostic pop -#endif +PYBIND11_WARNING_POP template struct type_caster::ValidType> { + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); using Helper = eigen_tensor_helper; static constexpr auto temp_name = get_tensor_descriptor::value; PYBIND11_TYPE_CASTER(Type, temp_name); @@ -185,7 +181,7 @@ struct type_caster::ValidType> { return false; } - if (!convert && !temp.dtype().is(dtype::of())) { + if (!temp.dtype().is(dtype::of())) { return false; } } @@ -288,7 +284,7 @@ struct type_caster::ValidType> { case return_value_policy::take_ownership: if (std::is_const::value) { // This cast is ugly, and might be UB in some cases, but we don't have an - // alterantive here as we must free that memory + // alternative here as we must free that memory Helper::free(const_cast(src)); pybind11_fail("Cannot take ownership of a const reference"); } @@ -366,6 +362,8 @@ struct get_storage_pointer_type struct type_caster, typename eigen_tensor_helper>::ValidType> { + static_assert(!std::is_pointer::value, + PYBIND11_EIGEN_MESSAGE_POINTER_TYPES_ARE_NOT_SUPPORTED); using MapType = Eigen::TensorMap; using Helper = eigen_tensor_helper>; @@ -389,7 +387,7 @@ struct type_caster, constexpr bool is_aligned = (Options & Eigen::Aligned) != 0; - if (PYBIND11_SILENCE_MSVC_C4127(is_aligned) && !is_tensor_aligned(arr.data())) { + if (is_aligned && !is_tensor_aligned(arr.data())) { return false; } @@ -399,7 +397,7 @@ struct type_caster, return false; } - if (PYBIND11_SILENCE_MSVC_C4127(needs_writeable) && !arr.writeable()) { + if (needs_writeable && !arr.writeable()) { return false; } diff --git a/lib/pybind11/include/pybind11/embed.h b/lib/pybind11/include/pybind11/embed.h index 0ac609e0..caa14f4a 100644 --- a/lib/pybind11/include/pybind11/embed.h +++ b/lib/pybind11/include/pybind11/embed.h @@ -86,38 +86,26 @@ inline wchar_t *widen_chars(const char *safe_arg) { return widened_arg; } -PYBIND11_NAMESPACE_END(detail) - -/** \rst - Initialize the Python interpreter. No other pybind11 or CPython API functions can be - called before this is done; with the exception of `PYBIND11_EMBEDDED_MODULE`. The - optional `init_signal_handlers` parameter can be used to skip the registration of - signal handlers (see the `Python documentation`_ for details). Calling this function - again after the interpreter has already been initialized is a fatal error. - - If initializing the Python interpreter fails, then the program is terminated. (This - is controlled by the CPython runtime and is an exception to pybind11's normal behavior - of throwing exceptions on errors.) - - The remaining optional parameters, `argc`, `argv`, and `add_program_dir_to_path` are - used to populate ``sys.argv`` and ``sys.path``. - See the |PySys_SetArgvEx documentation|_ for details. - - .. _Python documentation: https://docs.python.org/3/c-api/init.html#c.Py_InitializeEx - .. |PySys_SetArgvEx documentation| replace:: ``PySys_SetArgvEx`` documentation - .. _PySys_SetArgvEx documentation: https://docs.python.org/3/c-api/init.html#c.PySys_SetArgvEx - \endrst */ -inline void initialize_interpreter(bool init_signal_handlers = true, - int argc = 0, - const char *const *argv = nullptr, - bool add_program_dir_to_path = true) { +inline void precheck_interpreter() { if (Py_IsInitialized() != 0) { pybind11_fail("The interpreter is already running"); } +} -#if PY_VERSION_HEX < 0x030B0000 +#if !defined(PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX) +# define PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX (0x03080000) +#endif +#if PY_VERSION_HEX < PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX +inline void initialize_interpreter_pre_pyconfig(bool init_signal_handlers, + int argc, + const char *const *argv, + bool add_program_dir_to_path) { + detail::precheck_interpreter(); Py_InitializeEx(init_signal_handlers ? 1 : 0); +# if defined(WITH_THREAD) && PY_VERSION_HEX < 0x03070000 + PyEval_InitThreads(); +# endif // Before it was special-cased in python 3.8, passing an empty or null argv // caused a segfault, so we have to reimplement the special case ourselves. @@ -147,26 +135,30 @@ inline void initialize_interpreter(bool init_signal_handlers = true, auto *pysys_argv = widened_argv.get(); PySys_SetArgvEx(argc, pysys_argv, static_cast(add_program_dir_to_path)); -#else - PyConfig config; - PyConfig_InitIsolatedConfig(&config); - config.isolated = 0; - config.use_environment = 1; - config.install_signal_handlers = init_signal_handlers ? 1 : 0; +} +#endif - PyStatus status = PyConfig_SetBytesArgv(&config, argc, const_cast(argv)); - if (PyStatus_Exception(status)) { +PYBIND11_NAMESPACE_END(detail) + +#if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX +inline void initialize_interpreter(PyConfig *config, + int argc = 0, + const char *const *argv = nullptr, + bool add_program_dir_to_path = true) { + detail::precheck_interpreter(); + PyStatus status = PyConfig_SetBytesArgv(config, argc, const_cast(argv)); + if (PyStatus_Exception(status) != 0) { // A failure here indicates a character-encoding failure or the python // interpreter out of memory. Give up. - PyConfig_Clear(&config); - throw std::runtime_error(PyStatus_IsError(status) ? status.err_msg - : "Failed to prepare CPython"); + PyConfig_Clear(config); + throw std::runtime_error(PyStatus_IsError(status) != 0 ? status.err_msg + : "Failed to prepare CPython"); } - status = Py_InitializeFromConfig(&config); - PyConfig_Clear(&config); - if (PyStatus_Exception(status)) { - throw std::runtime_error(PyStatus_IsError(status) ? status.err_msg - : "Failed to init CPython"); + status = Py_InitializeFromConfig(config); + if (PyStatus_Exception(status) != 0) { + PyConfig_Clear(config); + throw std::runtime_error(PyStatus_IsError(status) != 0 ? status.err_msg + : "Failed to init CPython"); } if (add_program_dir_to_path) { PyRun_SimpleString("import sys, os.path; " @@ -174,6 +166,44 @@ inline void initialize_interpreter(bool init_signal_handlers = true, "os.path.abspath(os.path.dirname(sys.argv[0])) " "if sys.argv and os.path.exists(sys.argv[0]) else '')"); } + PyConfig_Clear(config); +} +#endif + +/** \rst + Initialize the Python interpreter. No other pybind11 or CPython API functions can be + called before this is done; with the exception of `PYBIND11_EMBEDDED_MODULE`. The + optional `init_signal_handlers` parameter can be used to skip the registration of + signal handlers (see the `Python documentation`_ for details). Calling this function + again after the interpreter has already been initialized is a fatal error. + + If initializing the Python interpreter fails, then the program is terminated. (This + is controlled by the CPython runtime and is an exception to pybind11's normal behavior + of throwing exceptions on errors.) + + The remaining optional parameters, `argc`, `argv`, and `add_program_dir_to_path` are + used to populate ``sys.argv`` and ``sys.path``. + See the |PySys_SetArgvEx documentation|_ for details. + + .. _Python documentation: https://docs.python.org/3/c-api/init.html#c.Py_InitializeEx + .. |PySys_SetArgvEx documentation| replace:: ``PySys_SetArgvEx`` documentation + .. _PySys_SetArgvEx documentation: https://docs.python.org/3/c-api/init.html#c.PySys_SetArgvEx + \endrst */ +inline void initialize_interpreter(bool init_signal_handlers = true, + int argc = 0, + const char *const *argv = nullptr, + bool add_program_dir_to_path = true) { +#if PY_VERSION_HEX < PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX + detail::initialize_interpreter_pre_pyconfig( + init_signal_handlers, argc, argv, add_program_dir_to_path); +#else + PyConfig config; + PyConfig_InitPythonConfig(&config); + // See PR #4473 for background + config.parse_argv = 0; + + config.install_signal_handlers = init_signal_handlers ? 1 : 0; + initialize_interpreter(&config, argc, argv, add_program_dir_to_path); #endif } @@ -213,16 +243,14 @@ inline void initialize_interpreter(bool init_signal_handlers = true, \endrst */ inline void finalize_interpreter() { - handle builtins(PyEval_GetBuiltins()); - const char *id = PYBIND11_INTERNALS_ID; - // Get the internals pointer (without creating it if it doesn't exist). It's possible for the // internals to be created during Py_Finalize() (e.g. if a py::capsule calls `get_internals()` // during destruction), so we get the pointer-pointer here and check it after Py_Finalize(). detail::internals **internals_ptr_ptr = detail::get_internals_pp(); - // It could also be stashed in builtins, so look there too: - if (builtins.contains(id) && isinstance(builtins[id])) { - internals_ptr_ptr = capsule(builtins[id]); + // It could also be stashed in state_dict, so look there too: + if (object internals_obj + = get_internals_obj_from_state_dict(detail::get_python_state_dict())) { + internals_ptr_ptr = detail::get_internals_pp_from_capsule(internals_obj); } // Local internals contains data managed by the current interpreter, so we must clear them to // avoid undefined behaviors when initializing another interpreter @@ -261,6 +289,15 @@ class scoped_interpreter { initialize_interpreter(init_signal_handlers, argc, argv, add_program_dir_to_path); } +#if PY_VERSION_HEX >= PYBIND11_PYCONFIG_SUPPORT_PY_VERSION_HEX + explicit scoped_interpreter(PyConfig *config, + int argc = 0, + const char *const *argv = nullptr, + bool add_program_dir_to_path = true) { + initialize_interpreter(config, argc, argv, add_program_dir_to_path); + } +#endif + scoped_interpreter(const scoped_interpreter &) = delete; scoped_interpreter(scoped_interpreter &&other) noexcept { other.is_valid = false; } scoped_interpreter &operator=(const scoped_interpreter &) = delete; diff --git a/lib/pybind11/include/pybind11/gil.h b/lib/pybind11/include/pybind11/gil.h index cb0028d5..da22f48d 100644 --- a/lib/pybind11/include/pybind11/gil.h +++ b/lib/pybind11/include/pybind11/gil.h @@ -11,6 +11,8 @@ #include "detail/common.h" +#include + #if defined(WITH_THREAD) && !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) # include "detail/internals.h" #endif @@ -137,7 +139,9 @@ class gil_scoped_acquire { class gil_scoped_release { public: + // PRECONDITION: The GIL must be held when this constructor is called. explicit gil_scoped_release(bool disassoc = false) : disassoc(disassoc) { + assert(PyGILState_Check()); // `get_internals()` must be called here unconditionally in order to initialize // `internals.tstate` for subsequent `gil_scoped_acquire` calls. Otherwise, an // initialization race could occur as multiple threads try `gil_scoped_acquire`. @@ -152,8 +156,8 @@ class gil_scoped_release { } } - gil_scoped_release(const gil_scoped_acquire &) = delete; - gil_scoped_release &operator=(const gil_scoped_acquire &) = delete; + gil_scoped_release(const gil_scoped_release &) = delete; + gil_scoped_release &operator=(const gil_scoped_release &) = delete; /// This method will disable the PyThreadState_DeleteCurrent call and the /// GIL won't be acquired. This method should be used if the interpreter @@ -201,9 +205,13 @@ class gil_scoped_release { PyThreadState *state; public: - gil_scoped_release() : state{PyEval_SaveThread()} {} + // PRECONDITION: The GIL must be held when this constructor is called. + gil_scoped_release() { + assert(PyGILState_Check()); + state = PyEval_SaveThread(); + } gil_scoped_release(const gil_scoped_release &) = delete; - gil_scoped_release &operator=(const gil_scoped_acquire &) = delete; + gil_scoped_release &operator=(const gil_scoped_release &) = delete; ~gil_scoped_release() { PyEval_RestoreThread(state); } void disarm() {} }; @@ -230,7 +238,7 @@ class gil_scoped_release { (void) (this != (this + 1)); } gil_scoped_release(const gil_scoped_release &) = delete; - gil_scoped_release &operator=(const gil_scoped_acquire &) = delete; + gil_scoped_release &operator=(const gil_scoped_release &) = delete; void disarm() {} }; diff --git a/lib/pybind11/include/pybind11/gil_safe_call_once.h b/lib/pybind11/include/pybind11/gil_safe_call_once.h new file mode 100644 index 00000000..eaf84d16 --- /dev/null +++ b/lib/pybind11/include/pybind11/gil_safe_call_once.h @@ -0,0 +1,91 @@ +// Copyright (c) 2023 The pybind Community. + +#pragma once + +#include "detail/common.h" +#include "gil.h" + +#include +#include + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + +// Use the `gil_safe_call_once_and_store` class below instead of the naive +// +// static auto imported_obj = py::module_::import("module_name"); // BAD, DO NOT USE! +// +// which has two serious issues: +// +// 1. Py_DECREF() calls potentially after the Python interpreter was finalized already, and +// 2. deadlocks in multi-threaded processes (because of missing lock ordering). +// +// The following alternative avoids both problems: +// +// PYBIND11_CONSTINIT static py::gil_safe_call_once_and_store storage; +// auto &imported_obj = storage // Do NOT make this `static`! +// .call_once_and_store_result([]() { +// return py::module_::import("module_name"); +// }) +// .get_stored(); +// +// The parameter of `call_once_and_store_result()` must be callable. It can make +// CPython API calls, and in particular, it can temporarily release the GIL. +// +// `T` can be any C++ type, it does not have to involve CPython API types. +// +// The behavior with regard to signals, e.g. `SIGINT` (`KeyboardInterrupt`), +// is not ideal. If the main thread is the one to actually run the `Callable`, +// then a `KeyboardInterrupt` will interrupt it if it is running normal Python +// code. The situation is different if a non-main thread runs the +// `Callable`, and then the main thread starts waiting for it to complete: +// a `KeyboardInterrupt` will not interrupt the non-main thread, but it will +// get processed only when it is the main thread's turn again and it is running +// normal Python code. However, this will be unnoticeable for quick call-once +// functions, which is usually the case. +template +class gil_safe_call_once_and_store { +public: + // PRECONDITION: The GIL must be held when `call_once_and_store_result()` is called. + template + gil_safe_call_once_and_store &call_once_and_store_result(Callable &&fn) { + if (!is_initialized_) { // This read is guarded by the GIL. + // Multiple threads may enter here, because the GIL is released in the next line and + // CPython API calls in the `fn()` call below may release and reacquire the GIL. + gil_scoped_release gil_rel; // Needed to establish lock ordering. + std::call_once(once_flag_, [&] { + // Only one thread will ever enter here. + gil_scoped_acquire gil_acq; + ::new (storage_) T(fn()); // fn may release, but will reacquire, the GIL. + is_initialized_ = true; // This write is guarded by the GIL. + }); + // All threads will observe `is_initialized_` as true here. + } + // Intentionally not returning `T &` to ensure the calling code is self-documenting. + return *this; + } + + // This must only be called after `call_once_and_store_result()` was called. + T &get_stored() { + assert(is_initialized_); + PYBIND11_WARNING_PUSH +#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ < 5 + // Needed for gcc 4.8.5 + PYBIND11_WARNING_DISABLE_GCC("-Wstrict-aliasing") +#endif + return *reinterpret_cast(storage_); + PYBIND11_WARNING_POP + } + + constexpr gil_safe_call_once_and_store() = default; + PYBIND11_DTOR_CONSTEXPR ~gil_safe_call_once_and_store() = default; + +private: + alignas(T) char storage_[sizeof(T)] = {}; + std::once_flag once_flag_ = {}; + bool is_initialized_ = false; + // The `is_initialized_`-`storage_` pair is very similar to `std::optional`, + // but the latter does not have the triviality properties of former, + // therefore `std::optional` is not a viable alternative here. +}; + +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/lib/pybind11/include/pybind11/numpy.h b/lib/pybind11/include/pybind11/numpy.h index 369e1864..8551aa26 100644 --- a/lib/pybind11/include/pybind11/numpy.h +++ b/lib/pybind11/include/pybind11/numpy.h @@ -10,7 +10,10 @@ #pragma once #include "pybind11.h" +#include "detail/common.h" #include "complex.h" +#include "gil_safe_call_once.h" +#include "pytypes.h" #include #include @@ -36,6 +39,8 @@ static_assert(std::is_signed::value, "Py_intptr_t must be signed"); PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_WARNING_DISABLE_MSVC(4127) + class array; // Forward declaration PYBIND11_NAMESPACE_BEGIN(detail) @@ -118,6 +123,20 @@ inline numpy_internals &get_numpy_internals() { return *ptr; } +PYBIND11_NOINLINE module_ import_numpy_core_submodule(const char *submodule_name) { + module_ numpy = module_::import("numpy"); + str version_string = numpy.attr("__version__"); + + module_ numpy_lib = module_::import("numpy.lib"); + object numpy_version = numpy_lib.attr("NumpyVersion")(version_string); + int major_version = numpy_version.attr("major").cast(); + + /* `numpy.core` was renamed to `numpy._core` in NumPy 2.0 as it officially + became a private module. */ + std::string numpy_core_path = major_version >= 2 ? "numpy._core" : "numpy.core"; + return module_::import((numpy_core_path + "." + submodule_name).c_str()); +} + template struct same_size { template @@ -190,8 +209,8 @@ struct npy_api { }; static npy_api &get() { - static npy_api api = lookup(); - return api; + PYBIND11_CONSTINIT static gil_safe_call_once_and_store storage; + return storage.call_once_and_store_result(lookup).get_stored(); } bool PyArray_Check_(PyObject *obj) const { @@ -261,9 +280,13 @@ struct npy_api { }; static npy_api lookup() { - module_ m = module_::import("numpy.core.multiarray"); + module_ m = detail::import_numpy_core_submodule("multiarray"); auto c = m.attr("_ARRAY_API"); void **api_ptr = (void **) PyCapsule_GetPointer(c.ptr(), nullptr); + if (api_ptr == nullptr) { + raise_from(PyExc_SystemError, "FAILURE obtaining numpy _ARRAY_API pointer."); + throw error_already_set(); + } npy_api api; #define DECL_NPY_API(Func) api.Func##_ = (decltype(api.Func##_)) api_ptr[API_##Func]; DECL_NPY_API(PyArray_GetNDArrayCFeatureVersion); @@ -562,6 +585,8 @@ class dtype : public object { m_ptr = from_args(args).release().ptr(); } + /// Return dtype for the given typenum (one of the NPY_TYPES). + /// https://numpy.org/devdocs/reference/c-api/array.html#c.PyArray_DescrFromType explicit dtype(int typenum) : object(detail::npy_api::get().PyArray_DescrFromType_(typenum), stolen_t{}) { if (m_ptr == nullptr) { @@ -621,13 +646,14 @@ class dtype : public object { char flags() const { return detail::array_descriptor_proxy(m_ptr)->flags; } private: - static object _dtype_from_pep3118() { - static PyObject *obj = module_::import("numpy.core._internal") - .attr("_dtype_from_pep3118") - .cast() - .release() - .ptr(); - return reinterpret_borrow(obj); + static object &_dtype_from_pep3118() { + PYBIND11_CONSTINIT static gil_safe_call_once_and_store storage; + return storage + .call_once_and_store_result([]() { + return detail::import_numpy_core_submodule("_internal") + .attr("_dtype_from_pep3118"); + }) + .get_stored(); } dtype strip_padding(ssize_t itemsize) { @@ -875,7 +901,7 @@ class array : public buffer { */ template detail::unchecked_mutable_reference mutable_unchecked() & { - if (PYBIND11_SILENCE_MSVC_C4127(Dims >= 0) && ndim() != Dims) { + if (Dims >= 0 && ndim() != Dims) { throw std::domain_error("array has incorrect number of dimensions: " + std::to_string(ndim()) + "; expected " + std::to_string(Dims)); @@ -893,7 +919,7 @@ class array : public buffer { */ template detail::unchecked_reference unchecked() const & { - if (PYBIND11_SILENCE_MSVC_C4127(Dims >= 0) && ndim() != Dims) { + if (Dims >= 0 && ndim() != Dims) { throw std::domain_error("array has incorrect number of dimensions: " + std::to_string(ndim()) + "; expected " + std::to_string(Dims)); @@ -1004,7 +1030,7 @@ class array : public buffer { /// Create array from any object -- always returns a new reference static PyObject *raw_array(PyObject *ptr, int ExtraFlags = 0) { if (ptr == nullptr) { - PyErr_SetString(PyExc_ValueError, "cannot create a pybind11::array from a nullptr"); + set_error(PyExc_ValueError, "cannot create a pybind11::array from a nullptr"); return nullptr; } return detail::npy_api::get().PyArray_FromAny_( @@ -1119,10 +1145,10 @@ class array_t : public array { /** * Returns a proxy object that provides const access to the array's data without bounds or - * dimensionality checking. Unlike `unchecked()`, this does not require that the underlying - * array have the `writable` flag. Use with care: the array must not be destroyed or reshaped - * for the duration of the returned object, and the caller must take care not to access invalid - * dimensions or dimension indices. + * dimensionality checking. Unlike `mutable_unchecked()`, this does not require that the + * underlying array have the `writable` flag. Use with care: the array must not be destroyed + * or reshaped for the duration of the returned object, and the caller must take care not to + * access invalid dimensions or dimension indices. */ template detail::unchecked_reference unchecked() const & { @@ -1151,7 +1177,7 @@ class array_t : public array { /// Create array from any object -- always returns a new reference static PyObject *raw_array_t(PyObject *ptr) { if (ptr == nullptr) { - PyErr_SetString(PyExc_ValueError, "cannot create a pybind11::array_t from a nullptr"); + set_error(PyExc_ValueError, "cannot create a pybind11::array_t from a nullptr"); return nullptr; } return detail::npy_api::get().PyArray_FromAny_(ptr, @@ -1281,12 +1307,16 @@ struct npy_format_descriptor< public: static constexpr int value = values[detail::is_fmt_numeric::index]; - static pybind11::dtype dtype() { - if (auto *ptr = npy_api::get().PyArray_DescrFromType_(value)) { - return reinterpret_steal(ptr); - } - pybind11_fail("Unsupported buffer format!"); - } + static pybind11::dtype dtype() { return pybind11::dtype(/*typenum*/ value); } +}; + +template +struct npy_format_descriptor::value>> { + static constexpr auto name = const_name("object"); + + static constexpr int value = npy_api::NPY_OBJECT_; + + static pybind11::dtype dtype() { return pybind11::dtype(/*typenum*/ value); } }; #define PYBIND11_DECL_CHAR_FMT \ @@ -1469,7 +1499,7 @@ struct npy_format_descriptor { } // Extract name, offset and format descriptor for a struct field -# define PYBIND11_FIELD_DESCRIPTOR(T, Field) PYBIND11_FIELD_DESCRIPTOR_EX(T, Field, # Field) +# define PYBIND11_FIELD_DESCRIPTOR(T, Field) PYBIND11_FIELD_DESCRIPTOR_EX(T, Field, #Field) // The main idea of this macro is borrowed from https://github.com/swansontec/map-macro // (C) William Swanson, Paul Fultz @@ -1865,9 +1895,10 @@ struct vectorize_helper { } auto result = returned_array::create(trivial, shape); + + PYBIND11_WARNING_PUSH #ifdef PYBIND11_DETECTED_CLANG_WITH_MISLEADING_CALL_STD_MOVE_EXPLICITLY_WARNING -# pragma clang diagnostic push -# pragma clang diagnostic ignored "-Wreturn-std-move" + PYBIND11_WARNING_DISABLE_CLANG("-Wreturn-std-move") #endif if (size == 0) { @@ -1883,9 +1914,7 @@ struct vectorize_helper { } return result; -#ifdef PYBIND11_DETECTED_CLANG_WITH_MISLEADING_CALL_STD_MOVE_EXPLICITLY_WARNING -# pragma clang diagnostic pop -#endif + PYBIND11_WARNING_POP } template diff --git a/lib/pybind11/include/pybind11/options.h b/lib/pybind11/include/pybind11/options.h index 1e493bdc..1b212252 100644 --- a/lib/pybind11/include/pybind11/options.h +++ b/lib/pybind11/include/pybind11/options.h @@ -47,6 +47,16 @@ class options { return *this; } + options &disable_enum_members_docstring() & { + global_state().show_enum_members_docstring = false; + return *this; + } + + options &enable_enum_members_docstring() & { + global_state().show_enum_members_docstring = true; + return *this; + } + // Getter methods (return the global state): static bool show_user_defined_docstrings() { @@ -55,6 +65,10 @@ class options { static bool show_function_signatures() { return global_state().show_function_signatures; } + static bool show_enum_members_docstring() { + return global_state().show_enum_members_docstring; + } + // This type is not meant to be allocated on the heap. void *operator new(size_t) = delete; @@ -63,6 +77,8 @@ class options { bool show_user_defined_docstrings = true; //< Include user-supplied texts in docstrings. bool show_function_signatures = true; //< Include auto-generated function signatures // in docstrings. + bool show_enum_members_docstring = true; //< Include auto-generated member list in enum + // docstrings. }; static state &global_state() { diff --git a/lib/pybind11/include/pybind11/pybind11.h b/lib/pybind11/include/pybind11/pybind11.h index 76d6eadc..b87fe66b 100644 --- a/lib/pybind11/include/pybind11/pybind11.h +++ b/lib/pybind11/include/pybind11/pybind11.h @@ -14,7 +14,9 @@ #include "detail/init.h" #include "attr.h" #include "gil.h" +#include "gil_safe_call_once.h" #include "options.h" +#include "typing.h" #include #include @@ -35,6 +37,8 @@ # include #endif +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) + /* https://stackoverflow.com/questions/46798456/handling-gccs-noexcept-type-warning This warning is about ABI compatibility, not code health. It is only actually needed in a couple places, but apparently GCC 7 "generates this warning if @@ -43,14 +47,54 @@ No other GCC version generates this warning. */ #if defined(__GNUC__) && __GNUC__ == 7 -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wnoexcept-type" +PYBIND11_WARNING_DISABLE_GCC("-Wnoexcept-type") #endif -PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_WARNING_DISABLE_MSVC(4127) PYBIND11_NAMESPACE_BEGIN(detail) +inline std::string replace_newlines_and_squash(const char *text) { + const char *whitespaces = " \t\n\r\f\v"; + std::string result(text); + bool previous_is_whitespace = false; + + if (result.size() >= 2) { + // Do not modify string representations + char first_char = result[0]; + char last_char = result[result.size() - 1]; + if (first_char == last_char && first_char == '\'') { + return result; + } + } + result.clear(); + + // Replace characters in whitespaces array with spaces and squash consecutive spaces + while (*text != '\0') { + if (std::strchr(whitespaces, *text)) { + if (!previous_is_whitespace) { + result += ' '; + previous_is_whitespace = true; + } + } else { + result += *text; + previous_is_whitespace = false; + } + ++text; + } + + // Strip leading and trailing whitespaces + const size_t str_begin = result.find_first_not_of(whitespaces); + if (str_begin == std::string::npos) { + return ""; + } + + const size_t str_end = result.find_last_not_of(whitespaces); + const size_t str_range = str_end - str_begin + 1; + + return result.substr(str_begin, str_range); +} + // Apply all the extensions translators from a list // Return true if one of the translators completed without raising an exception // itself. Return of false indicates that if there are other translators @@ -83,6 +127,7 @@ class cpp_function : public function { cpp_function() = default; // NOLINTNEXTLINE(google-explicit-constructor) cpp_function(std::nullptr_t) {} + cpp_function(std::nullptr_t, const is_setter &) {} /// Construct a cpp_function from a vanilla function pointer template @@ -177,22 +222,22 @@ class cpp_function : public function { auto *rec = unique_rec.get(); /* Store the capture object directly in the function record if there is enough space */ - if (PYBIND11_SILENCE_MSVC_C4127(sizeof(capture) <= sizeof(rec->data))) { + if (sizeof(capture) <= sizeof(rec->data)) { /* Without these pragmas, GCC warns that there might not be enough space to use the placement new operator. However, the 'if' statement above ensures that this is the case. */ -#if defined(__GNUG__) && __GNUC__ >= 6 && !defined(__clang__) && !defined(__INTEL_COMPILER) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wplacement-new" + PYBIND11_WARNING_PUSH + +#if defined(__GNUG__) && __GNUC__ >= 6 + PYBIND11_WARNING_DISABLE_GCC("-Wplacement-new") #endif + new ((capture *) &rec->data) capture{std::forward(f)}; -#if defined(__GNUG__) && __GNUC__ >= 6 && !defined(__clang__) && !defined(__INTEL_COMPILER) -# pragma GCC diagnostic pop -#endif -#if defined(__GNUG__) && !PYBIND11_HAS_STD_LAUNDER && !defined(__INTEL_COMPILER) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wstrict-aliasing" + +#if !PYBIND11_HAS_STD_LAUNDER + PYBIND11_WARNING_DISABLE_GCC("-Wstrict-aliasing") #endif + // UB without std::launder, but without breaking ABI and/or // a significant refactoring it's "impossible" to solve. if (!std::is_trivially_destructible::value) { @@ -202,9 +247,7 @@ class cpp_function : public function { data->~capture(); }; } -#if defined(__GNUG__) && !PYBIND11_HAS_STD_LAUNDER && !defined(__INTEL_COMPILER) -# pragma GCC diagnostic pop -#endif + PYBIND11_WARNING_POP } else { rec->data[0] = new capture{std::forward(f)}; rec->free_data = [](function_record *r) { delete ((capture *) r->data[0]); }; @@ -245,10 +288,16 @@ class cpp_function : public function { using Guard = extract_guard_t; /* Perform the function call */ - handle result - = cast_out::cast(std::move(args_converter).template call(cap->f), - policy, - call.parent); + handle result; + if (call.func.is_setter) { + (void) std::move(args_converter).template call(cap->f); + result = none().release(); + } else { + result = cast_out::cast( + std::move(args_converter).template call(cap->f), + policy, + call.parent); + } /* Invoke call policy post-call hook */ process_attributes::postcall(call, result); @@ -418,7 +467,7 @@ class cpp_function : public function { // Write default value if available. if (!is_starred && arg_index < rec->args.size() && rec->args[arg_index].descr) { signature += " = "; - signature += rec->args[arg_index].descr; + signature += detail::replace_newlines_and_squash(rec->args[arg_index].descr); } // Separator for positional-only arguments (placed after the // argument, rather than before like * @@ -502,8 +551,8 @@ class cpp_function : public function { rec->def->ml_flags = METH_VARARGS | METH_KEYWORDS; capsule rec_capsule(unique_rec.release(), + detail::get_function_record_capsule_name(), [](void *ptr) { destruct((detail::function_record *) ptr); }); - rec_capsule.set_name(detail::get_function_record_capsule_name()); guarded_strdup.release(); object scope_module; @@ -674,7 +723,7 @@ class cpp_function : public function { /* Iterator over the list of potentially admissible overloads */ const function_record *overloads = reinterpret_cast( PyCapsule_GetPointer(self, get_function_record_capsule_name())), - *it = overloads; + *current_overload = overloads; assert(overloads != nullptr); /* Need to know how many arguments + keyword arguments there are to pick the right @@ -688,9 +737,8 @@ class cpp_function : public function { if (overloads->is_constructor) { if (!parent || !PyObject_TypeCheck(parent.ptr(), (PyTypeObject *) overloads->scope.ptr())) { - PyErr_SetString( - PyExc_TypeError, - "__init__(self, ...) called with invalid or missing `self` argument"); + set_error(PyExc_TypeError, + "__init__(self, ...) called with invalid or missing `self` argument"); return nullptr; } @@ -713,9 +761,10 @@ class cpp_function : public function { std::vector second_pass; // However, if there are no overloads, we can just skip the no-convert pass entirely - const bool overloaded = it != nullptr && it->next != nullptr; + const bool overloaded + = current_overload != nullptr && current_overload->next != nullptr; - for (; it != nullptr; it = it->next) { + for (; current_overload != nullptr; current_overload = current_overload->next) { /* For each overload: 1. Copy all positional arguments we were given, also checking to make sure that @@ -736,7 +785,7 @@ class cpp_function : public function { a result other than PYBIND11_TRY_NEXT_OVERLOAD. */ - const function_record &func = *it; + const function_record &func = *current_overload; size_t num_args = func.nargs; // Number of positional arguments that we need if (func.has_args) { --num_args; // (but don't count py::args @@ -974,10 +1023,10 @@ class cpp_function : public function { } if (result.ptr() != PYBIND11_TRY_NEXT_OVERLOAD) { - // The error reporting logic below expects 'it' to be valid, as it would be - // if we'd encountered this failure in the first-pass loop. + // The error reporting logic below expects 'current_overload' to be valid, + // as it would be if we'd encountered this failure in the first-pass loop. if (!result) { - it = &call.func; + current_overload = &call.func; } break; } @@ -1001,7 +1050,7 @@ class cpp_function : public function { A translator may choose to do one of the following: - - catch the exception and call PyErr_SetString or PyErr_SetObject + - catch the exception and call py::set_error() to set a standard (or custom) Python exception, or - do nothing and let the exception fall through to the next translator, or - delegate translation to the next translator by throwing a new type of exception. @@ -1017,8 +1066,7 @@ class cpp_function : public function { return nullptr; } - PyErr_SetString(PyExc_SystemError, - "Exception escaped from default exception translator!"); + set_error(PyExc_SystemError, "Exception escaped from default exception translator!"); return nullptr; } @@ -1096,7 +1144,7 @@ class cpp_function : public function { } msg += "kwargs: "; bool first = true; - for (auto kwarg : kwargs) { + for (const auto &kwarg : kwargs) { if (first) { first = false; } else { @@ -1119,20 +1167,21 @@ class cpp_function : public function { raise_from(PyExc_TypeError, msg.c_str()); return nullptr; } - PyErr_SetString(PyExc_TypeError, msg.c_str()); + set_error(PyExc_TypeError, msg.c_str()); return nullptr; } if (!result) { std::string msg = "Unable to convert function return value to a " "Python type! The signature was\n\t"; - msg += it->signature; + assert(current_overload != nullptr); + msg += current_overload->signature; append_note_if_missing_header_is_suspected(msg); // Attach additional error info to the exception if supported if (PyErr_Occurred()) { raise_from(PyExc_TypeError, msg.c_str()); return nullptr; } - PyErr_SetString(PyExc_TypeError, msg.c_str()); + set_error(PyExc_TypeError, msg.c_str()); return nullptr; } if (overloads->is_constructor && !self_value_and_holder.holder_constructed()) { @@ -1427,9 +1476,9 @@ template ::value, int> = 0> void call_operator_delete(T *p, size_t, size_t) { T::operator delete(p); } -template < - typename T, - enable_if_t::value && has_operator_delete_size::value, int> = 0> +template ::value && has_operator_delete_size::value, int> + = 0> void call_operator_delete(T *p, size_t s, size_t) { T::operator delete(p, s); } @@ -1730,7 +1779,8 @@ class class_ : public detail::generic_type { template class_ & def_property(const char *name, const Getter &fget, const Setter &fset, const Extra &...extra) { - return def_property(name, fget, cpp_function(method_adaptor(fset)), extra...); + return def_property( + name, fget, cpp_function(method_adaptor(fset), is_setter()), extra...); } template class_ &def_property(const char *name, @@ -1841,8 +1891,7 @@ class class_ : public detail::generic_type { if (holder_ptr) { init_holder_from_existing(v_h, holder_ptr, std::is_copy_constructible()); v_h.set_holder_constructed(); - } else if (PYBIND11_SILENCE_MSVC_C4127(detail::always_construct_holder::value) - || inst->owned) { + } else if (detail::always_construct_holder::value || inst->owned) { new (std::addressof(v_h.holder())) holder_type(v_h.value_ptr()); v_h.set_holder_constructed(); } @@ -1971,32 +2020,38 @@ struct enum_base { object type_name = type::handle_of(arg).attr("__name__"); return pybind11::str("{}.{}").format(std::move(type_name), enum_name(arg)); }, - name("name"), + name("__str__"), is_method(m_base)); - m_base.attr("__doc__") = static_property( - cpp_function( - [](handle arg) -> std::string { - std::string docstring; - dict entries = arg.attr("__entries"); - if (((PyTypeObject *) arg.ptr())->tp_doc) { - docstring += std::string(((PyTypeObject *) arg.ptr())->tp_doc) + "\n\n"; - } - docstring += "Members:"; - for (auto kv : entries) { - auto key = std::string(pybind11::str(kv.first)); - auto comment = kv.second[int_(1)]; - docstring += "\n\n " + key; - if (!comment.is_none()) { - docstring += " : " + (std::string) pybind11::str(comment); + if (options::show_enum_members_docstring()) { + m_base.attr("__doc__") = static_property( + cpp_function( + [](handle arg) -> std::string { + std::string docstring; + dict entries = arg.attr("__entries"); + if (((PyTypeObject *) arg.ptr())->tp_doc) { + docstring += std::string( + reinterpret_cast(arg.ptr())->tp_doc); + docstring += "\n\n"; } - } - return docstring; - }, - name("__doc__")), - none(), - none(), - ""); + docstring += "Members:"; + for (auto kv : entries) { + auto key = std::string(pybind11::str(kv.first)); + auto comment = kv.second[int_(1)]; + docstring += "\n\n "; + docstring += key; + if (!comment.is_none()) { + docstring += " : "; + docstring += pybind11::str(comment).cast(); + } + } + return docstring; + }, + name("__doc__")), + none(), + none(), + ""); + } m_base.attr("__members__") = static_property(cpp_function( [](handle arg) -> dict { @@ -2383,7 +2438,7 @@ iterator make_iterator_impl(Iterator first, Sentinel last, Extra &&...extra) { Policy); } - return cast(state{first, last, true}); + return cast(state{std::forward(first), std::forward(last), true}); } PYBIND11_NAMESPACE_END(detail) @@ -2394,13 +2449,15 @@ template ::result_type, typename... Extra> -iterator make_iterator(Iterator first, Sentinel last, Extra &&...extra) { +typing::Iterator make_iterator(Iterator first, Sentinel last, Extra &&...extra) { return detail::make_iterator_impl, Policy, Iterator, Sentinel, ValueType, - Extra...>(first, last, std::forward(extra)...); + Extra...>(std::forward(first), + std::forward(last), + std::forward(extra)...); } /// Makes a python iterator over the keys (`.first`) of a iterator over pairs from a @@ -2410,13 +2467,15 @@ template ::result_type, typename... Extra> -iterator make_key_iterator(Iterator first, Sentinel last, Extra &&...extra) { +typing::Iterator make_key_iterator(Iterator first, Sentinel last, Extra &&...extra) { return detail::make_iterator_impl, Policy, Iterator, Sentinel, KeyType, - Extra...>(first, last, std::forward(extra)...); + Extra...>(std::forward(first), + std::forward(last), + std::forward(extra)...); } /// Makes a python iterator over the values (`.second`) of a iterator over pairs from a @@ -2426,21 +2485,25 @@ template ::result_type, typename... Extra> -iterator make_value_iterator(Iterator first, Sentinel last, Extra &&...extra) { +typing::Iterator make_value_iterator(Iterator first, Sentinel last, Extra &&...extra) { return detail::make_iterator_impl, Policy, Iterator, Sentinel, ValueType, - Extra...>(first, last, std::forward(extra)...); + Extra...>(std::forward(first), + std::forward(last), + std::forward(extra)...); } /// Makes an iterator over values of an stl container or other container supporting /// `std::begin()`/`std::end()` template ()))>::result_type, typename... Extra> -iterator make_iterator(Type &value, Extra &&...extra) { +typing::Iterator make_iterator(Type &value, Extra &&...extra) { return make_iterator( std::begin(value), std::end(value), std::forward(extra)...); } @@ -2449,8 +2512,10 @@ iterator make_iterator(Type &value, Extra &&...extra) { /// `std::begin()`/`std::end()` template ()))>::result_type, typename... Extra> -iterator make_key_iterator(Type &value, Extra &&...extra) { +typing::Iterator make_key_iterator(Type &value, Extra &&...extra) { return make_key_iterator( std::begin(value), std::end(value), std::forward(extra)...); } @@ -2459,8 +2524,10 @@ iterator make_key_iterator(Type &value, Extra &&...extra) { /// `std::begin()`/`std::end()` template ()))>::result_type, typename... Extra> -iterator make_value_iterator(Type &value, Extra &&...extra) { +typing::Iterator make_value_iterator(Type &value, Extra &&...extra) { return make_value_iterator( std::begin(value), std::end(value), std::forward(extra)...); } @@ -2516,7 +2583,7 @@ inline void register_local_exception_translator(ExceptionTranslator &&translator /** * Wrapper to generate a new Python exception type. * - * This should only be used with PyErr_SetString for now. + * This should only be used with py::set_error() for now. * It is not (yet) possible to use as a py::base. * Template type argument is reserved for future use. */ @@ -2537,27 +2604,20 @@ class exception : public object { } // Sets the current python exception to this exception object with the given message - void operator()(const char *message) { PyErr_SetString(m_ptr, message); } + PYBIND11_DEPRECATED("Please use py::set_error() instead " + "(https://github.com/pybind/pybind11/pull/4772)") + void operator()(const char *message) const { set_error(*this, message); } }; PYBIND11_NAMESPACE_BEGIN(detail) -// Returns a reference to a function-local static exception object used in the simple -// register_exception approach below. (It would be simpler to have the static local variable -// directly in register_exception, but that makes clang <3.5 segfault - issue #1349). -template -exception &get_exception_object() { - static exception ex; - return ex; -} // Helper function for register_exception and register_local_exception template exception & register_exception_impl(handle scope, const char *name, handle base, bool isLocal) { - auto &ex = detail::get_exception_object(); - if (!ex) { - ex = exception(scope, name, base); - } + PYBIND11_CONSTINIT static gil_safe_call_once_and_store> exc_storage; + exc_storage.call_once_and_store_result( + [&]() { return exception(scope, name, base); }); auto register_func = isLocal ? ®ister_local_exception_translator : ®ister_exception_translator; @@ -2569,10 +2629,10 @@ register_exception_impl(handle scope, const char *name, handle base, bool isLoca try { std::rethrow_exception(p); } catch (const CppException &e) { - detail::get_exception_object()(e.what()); + set_error(exc_storage.get_stored(), e.what()); } }); - return ex; + return exc_storage.get_stored(); } PYBIND11_NAMESPACE_END(detail) @@ -2691,7 +2751,11 @@ get_type_override(const void *this_ptr, const type_info *this_type, const char * if ((std::string) str(f_code->co_name) == name && f_code->co_argcount > 0) { PyObject *locals = PyEval_GetLocals(); if (locals != nullptr) { +# if PY_VERSION_HEX >= 0x030b0000 + PyObject *co_varnames = PyCode_GetVarnames(f_code); +# else PyObject *co_varnames = PyObject_GetAttrString((PyObject *) f_code, "co_varnames"); +# endif PyObject *self_arg = PyTuple_GET_ITEM(co_varnames, 0); Py_DECREF(co_varnames); PyObject *self_caller = dict_getitem(locals, self_arg); @@ -2876,7 +2940,3 @@ inline function get_overload(const T *this_ptr, const char *name) { PYBIND11_OVERRIDE_PURE(PYBIND11_TYPE(ret_type), PYBIND11_TYPE(cname), fn, __VA_ARGS__); PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) - -#if defined(__GNUC__) && __GNUC__ == 7 -# pragma GCC diagnostic pop // -Wnoexcept-type -#endif diff --git a/lib/pybind11/include/pybind11/pytypes.h b/lib/pybind11/include/pybind11/pytypes.h index 80b49ec3..e7c7c812 100644 --- a/lib/pybind11/include/pybind11/pytypes.h +++ b/lib/pybind11/include/pybind11/pytypes.h @@ -33,6 +33,8 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_WARNING_DISABLE_MSVC(4127) + /* A few forward declarations */ class handle; class object; @@ -230,7 +232,8 @@ class handle : public detail::object_api { detail::enable_if_t, detail::is_pyobj_ptr_or_nullptr_t>, std::is_convertible>::value, - int> = 0> + int> + = 0> // NOLINTNEXTLINE(google-explicit-constructor) handle(T &obj) : m_ptr(obj) {} @@ -246,6 +249,11 @@ class handle : public detail::object_api { const handle &inc_ref() const & { #ifdef PYBIND11_HANDLE_REF_DEBUG inc_ref_counter(1); +#endif +#ifdef PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF + if (m_ptr != nullptr && !PyGILState_Check()) { + throw_gilstate_error("pybind11::handle::inc_ref()"); + } #endif Py_XINCREF(m_ptr); return *this; @@ -257,6 +265,11 @@ class handle : public detail::object_api { this function automatically. Returns a reference to itself. \endrst */ const handle &dec_ref() const & { +#ifdef PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF + if (m_ptr != nullptr && !PyGILState_Check()) { + throw_gilstate_error("pybind11::handle::dec_ref()"); + } +#endif Py_XDECREF(m_ptr); return *this; } @@ -283,8 +296,33 @@ class handle : public detail::object_api { protected: PyObject *m_ptr = nullptr; -#ifdef PYBIND11_HANDLE_REF_DEBUG private: +#ifdef PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF + void throw_gilstate_error(const std::string &function_name) const { + fprintf( + stderr, + "%s is being called while the GIL is either not held or invalid. Please see " + "https://pybind11.readthedocs.io/en/stable/advanced/" + "misc.html#common-sources-of-global-interpreter-lock-errors for debugging advice.\n" + "If you are convinced there is no bug in your code, you can #define " + "PYBIND11_NO_ASSERT_GIL_HELD_INCREF_DECREF " + "to disable this check. In that case you have to ensure this #define is consistently " + "used for all translation units linked into a given pybind11 extension, otherwise " + "there will be ODR violations.", + function_name.c_str()); + if (Py_TYPE(m_ptr)->tp_name != nullptr) { + fprintf(stderr, + " The failing %s call was triggered on a %s object.", + function_name.c_str(), + Py_TYPE(m_ptr)->tp_name); + } + fprintf(stderr, "\n"); + fflush(stderr); + throw std::runtime_error(function_name + " PyGILState_Check() failure."); + } +#endif + +#ifdef PYBIND11_HANDLE_REF_DEBUG static std::size_t inc_ref_counter(std::size_t add) { thread_local std::size_t counter = 0; counter += add; @@ -296,6 +334,14 @@ class handle : public detail::object_api { #endif }; +inline void set_error(const handle &type, const char *message) { + PyErr_SetString(type.ptr(), message); +} + +inline void set_error(const handle &type, const handle &value) { + PyErr_SetObject(type.ptr(), value.ptr()); +} + /** \rst Holds a reference to a Python object (with reference counting) @@ -430,7 +476,7 @@ PYBIND11_NAMESPACE_BEGIN(detail) // Equivalent to obj.__class__.__name__ (or obj.__name__ if obj is a class). inline const char *obj_class_name(PyObject *obj) { - if (Py_TYPE(obj) == &PyType_Type) { + if (PyType_Check(obj)) { return reinterpret_cast(obj)->tp_name; } return Py_TYPE(obj)->tp_name; @@ -438,13 +484,24 @@ inline const char *obj_class_name(PyObject *obj) { std::string error_string(); +// The code in this struct is very unusual, to minimize the chances of +// masking bugs (elsewhere) by errors during the error handling (here). +// This is meant to be a lifeline for troubleshooting long-running processes +// that crash under conditions that are virtually impossible to reproduce. +// Low-level implementation alternatives are preferred to higher-level ones +// that might raise cascading exceptions. Last-ditch-kind-of attempts are made +// to report as much of the original error as possible, even if there are +// secondary issues obtaining some of the details. struct error_fetch_and_normalize { - // Immediate normalization is long-established behavior (starting with - // https://github.com/pybind/pybind11/commit/135ba8deafb8bf64a15b24d1513899eb600e2011 - // from Sep 2016) and safest. Normalization could be deferred, but this could mask - // errors elsewhere, the performance gain is very minor in typical situations - // (usually the dominant bottleneck is EH unwinding), and the implementation here - // would be more complex. + // This comment only applies to Python <= 3.11: + // Immediate normalization is long-established behavior (starting with + // https://github.com/pybind/pybind11/commit/135ba8deafb8bf64a15b24d1513899eb600e2011 + // from Sep 2016) and safest. Normalization could be deferred, but this could mask + // errors elsewhere, the performance gain is very minor in typical situations + // (usually the dominant bottleneck is EH unwinding), and the implementation here + // would be more complex. + // Starting with Python 3.12, PyErr_Fetch() normalizes exceptions immediately. + // Any errors during normalization are tracked under __notes__. explicit error_fetch_and_normalize(const char *called) { PyErr_Fetch(&m_type.ptr(), &m_value.ptr(), &m_trace.ptr()); if (!m_type) { @@ -459,6 +516,14 @@ struct error_fetch_and_normalize { "of the original active exception type."); } m_lazy_error_string = exc_type_name_orig; +#if PY_VERSION_HEX >= 0x030C0000 + // The presence of __notes__ is likely due to exception normalization + // errors, although that is not necessarily true, therefore insert a + // hint only: + if (PyObject_HasAttrString(m_value.ptr(), "__notes__")) { + m_lazy_error_string += "[WITH __notes__]"; + } +#else // PyErr_NormalizeException() may change the exception type if there are cascading // failures. This can potentially be extremely confusing. PyErr_NormalizeException(&m_type.ptr(), &m_value.ptr(), &m_trace.ptr()); @@ -468,17 +533,17 @@ struct error_fetch_and_normalize { "active exception."); } const char *exc_type_name_norm = detail::obj_class_name(m_type.ptr()); - if (exc_type_name_orig == nullptr) { + if (exc_type_name_norm == nullptr) { pybind11_fail("Internal error: " + std::string(called) + " failed to obtain the name " "of the normalized active exception type."); } -#if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x07030a00 +# if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x07030a00 // This behavior runs the risk of masking errors in the error handling, but avoids a // conflict with PyPy, which relies on the normalization here to change OSError to // FileNotFoundError (https://github.com/pybind/pybind11/issues/4075). m_lazy_error_string = exc_type_name_norm; -#else +# else if (exc_type_name_norm != m_lazy_error_string) { std::string msg = std::string(called) + ": MISMATCH of original and normalized " @@ -490,6 +555,7 @@ struct error_fetch_and_normalize { msg += ": " + format_value_and_trace(); pybind11_fail(msg); } +# endif #endif } @@ -525,6 +591,40 @@ struct error_fetch_and_normalize { } } } +#if PY_VERSION_HEX >= 0x030B0000 + auto notes + = reinterpret_steal(PyObject_GetAttrString(m_value.ptr(), "__notes__")); + if (!notes) { + PyErr_Clear(); // No notes is good news. + } else { + auto len_notes = PyList_Size(notes.ptr()); + if (len_notes < 0) { + result += "\nFAILURE obtaining len(__notes__): " + detail::error_string(); + } else { + result += "\n__notes__ (len=" + std::to_string(len_notes) + "):"; + for (ssize_t i = 0; i < len_notes; i++) { + PyObject *note = PyList_GET_ITEM(notes.ptr(), i); + auto note_bytes = reinterpret_steal( + PyUnicode_AsEncodedString(note, "utf-8", "backslashreplace")); + if (!note_bytes) { + result += "\nFAILURE obtaining __notes__[" + std::to_string(i) + + "]: " + detail::error_string(); + } else { + char *buffer = nullptr; + Py_ssize_t length = 0; + if (PyBytes_AsStringAndSize(note_bytes.ptr(), &buffer, &length) + == -1) { + result += "\nFAILURE formatting __notes__[" + std::to_string(i) + + "]: " + detail::error_string(); + } else { + result += '\n'; + result += std::string(buffer, static_cast(length)); + } + } + } + } + } +#endif } else { result = ""; } @@ -884,10 +984,8 @@ object object_or_cast(T &&o); // Match a PyObject*, which we want to convert directly to handle via its converting constructor inline handle object_or_cast(PyObject *ptr) { return ptr; } -#if defined(_MSC_VER) && _MSC_VER < 1920 -# pragma warning(push) -# pragma warning(disable : 4522) // warning C4522: multiple assignment operators specified -#endif +PYBIND11_WARNING_PUSH +PYBIND11_WARNING_DISABLE_MSVC(4522) // warning C4522: multiple assignment operators specified template class accessor : public object_api> { using key_type = typename Policy::key_type; @@ -951,9 +1049,7 @@ class accessor : public object_api> { key_type key; mutable object cache; }; -#if defined(_MSC_VER) && _MSC_VER < 1920 -# pragma warning(pop) -#endif +PYBIND11_WARNING_POP PYBIND11_NAMESPACE_BEGIN(accessor_policies) struct obj_attr { @@ -1524,7 +1620,15 @@ inline namespace literals { /** \rst String literal version of `str` \endrst */ -inline str operator"" _s(const char *s, size_t size) { return {s, size}; } +inline str +#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ < 5 +operator"" _s // gcc 4.8.5 insists on having a space (hard error). +#else +operator""_s // clang 17 generates a deprecation warning if there is a space. +#endif + (const char *s, size_t size) { + return {s, size}; +} } // namespace literals /// \addtogroup pytypes @@ -1693,7 +1797,7 @@ PYBIND11_NAMESPACE_BEGIN(detail) // unsigned type: (A)-1 != (B)-1 when A and B are unsigned types of different sizes). template Unsigned as_unsigned(PyObject *o) { - if (PYBIND11_SILENCE_MSVC_C4127(sizeof(Unsigned) <= sizeof(unsigned long))) { + if (sizeof(Unsigned) <= sizeof(unsigned long)) { unsigned long v = PyLong_AsUnsignedLong(o); return v == (unsigned long) -1 && PyErr_Occurred() ? (Unsigned) -1 : (Unsigned) v; } @@ -1710,7 +1814,7 @@ class int_ : public object { template ::value, int> = 0> // NOLINTNEXTLINE(google-explicit-constructor) int_(T value) { - if (PYBIND11_SILENCE_MSVC_C4127(sizeof(T) <= sizeof(long))) { + if (sizeof(T) <= sizeof(long)) { if (std::is_signed::value) { m_ptr = PyLong_FromLong((long) value); } else { @@ -1842,28 +1946,13 @@ class capsule : public object { } } + /// Capsule name is nullptr. capsule(const void *value, void (*destructor)(void *)) { - m_ptr = PyCapsule_New(const_cast(value), nullptr, [](PyObject *o) { - // guard if destructor called while err indicator is set - error_scope error_guard; - auto destructor = reinterpret_cast(PyCapsule_GetContext(o)); - if (destructor == nullptr && PyErr_Occurred()) { - throw error_already_set(); - } - const char *name = get_name_in_error_scope(o); - void *ptr = PyCapsule_GetPointer(o, name); - if (ptr == nullptr) { - throw error_already_set(); - } - - if (destructor != nullptr) { - destructor(ptr); - } - }); + initialize_with_void_ptr_destructor(value, nullptr, destructor); + } - if (!m_ptr || PyCapsule_SetContext(m_ptr, reinterpret_cast(destructor)) != 0) { - throw error_already_set(); - } + capsule(const void *value, const char *name, void (*destructor)(void *)) { + initialize_with_void_ptr_destructor(value, name, destructor); } explicit capsule(void (*destructor)()) { @@ -1931,6 +2020,32 @@ class capsule : public object { return name; } + + void initialize_with_void_ptr_destructor(const void *value, + const char *name, + void (*destructor)(void *)) { + m_ptr = PyCapsule_New(const_cast(value), name, [](PyObject *o) { + // guard if destructor called while err indicator is set + error_scope error_guard; + auto destructor = reinterpret_cast(PyCapsule_GetContext(o)); + if (destructor == nullptr && PyErr_Occurred()) { + throw error_already_set(); + } + const char *name = get_name_in_error_scope(o); + void *ptr = PyCapsule_GetPointer(o, name); + if (ptr == nullptr) { + throw error_already_set(); + } + + if (destructor != nullptr) { + destructor(ptr); + } + }); + + if (!m_ptr || PyCapsule_SetContext(m_ptr, reinterpret_cast(destructor)) != 0) { + throw error_already_set(); + } + } }; class tuple : public object { diff --git a/lib/pybind11/include/pybind11/stl.h b/lib/pybind11/include/pybind11/stl.h index 2d144b59..f0334b8f 100644 --- a/lib/pybind11/include/pybind11/stl.h +++ b/lib/pybind11/include/pybind11/stl.h @@ -100,7 +100,7 @@ struct set_caster { return s.release(); } - PYBIND11_TYPE_CASTER(type, const_name("Set[") + key_conv::name + const_name("]")); + PYBIND11_TYPE_CASTER(type, const_name("set[") + key_conv::name + const_name("]")); }; template @@ -157,7 +157,7 @@ struct map_caster { } PYBIND11_TYPE_CASTER(Type, - const_name("Dict[") + key_conv::name + const_name(", ") + value_conv::name + const_name("dict[") + key_conv::name + const_name(", ") + value_conv::name + const_name("]")); }; @@ -172,7 +172,7 @@ struct list_caster { auto s = reinterpret_borrow(src); value.clear(); reserve_maybe(s, &value); - for (auto it : s) { + for (const auto &it : s) { value_conv conv; if (!conv.load(it, convert)) { return false; @@ -208,7 +208,7 @@ struct list_caster { return l.release(); } - PYBIND11_TYPE_CASTER(Type, const_name("List[") + value_conv::name + const_name("]")); + PYBIND11_TYPE_CASTER(Type, const_name("list[") + value_conv::name + const_name("]")); }; template @@ -247,7 +247,7 @@ struct array_caster { return false; } size_t ctr = 0; - for (auto it : l) { + for (const auto &it : l) { value_conv conv; if (!conv.load(it, convert)) { return false; @@ -273,11 +273,11 @@ struct array_caster { } PYBIND11_TYPE_CASTER(ArrayType, - const_name("List[") + value_conv::name + const_name(const_name(""), const_name("Annotated[")) + + const_name("list[") + value_conv::name + const_name("]") + const_name(const_name(""), - const_name("[") + const_name() - + const_name("]")) - + const_name("]")); + const_name(", FixedSize(") + + const_name() + const_name(")]"))); }; template @@ -316,6 +316,7 @@ struct optional_caster { if (!std::is_lvalue_reference::value) { policy = return_value_policy_override::policy(policy); } + // NOLINTNEXTLINE(bugprone-unchecked-optional-access) return value_conv::cast(*std::forward(src), policy, parent); } diff --git a/lib/pybind11/include/pybind11/stl_bind.h b/lib/pybind11/include/pybind11/stl_bind.h index 22a29b47..4d0854f6 100644 --- a/lib/pybind11/include/pybind11/stl_bind.h +++ b/lib/pybind11/include/pybind11/stl_bind.h @@ -10,10 +10,13 @@ #pragma once #include "detail/common.h" +#include "detail/type_caster_base.h" +#include "cast.h" #include "operators.h" #include #include +#include PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) @@ -58,9 +61,11 @@ struct is_comparable< /* For a vector/map data structure, recursively check the value type (which is std::pair for maps) */ template -struct is_comparable::is_vector>> { - static constexpr const bool value = is_comparable::value; -}; +struct is_comparable::is_vector>> + : is_comparable::type_to_check_recursively> {}; + +template <> +struct is_comparable : std::true_type {}; /* For pairs, recursively check the two data types */ template @@ -352,13 +357,17 @@ void vector_accessor(enable_if_t::value, Class_> &cl) using DiffType = typename Vector::difference_type; using ItType = typename Vector::iterator; cl.def("__getitem__", [](const Vector &v, DiffType i) -> T { - if (i < 0 && (i += v.size()) < 0) { - throw index_error(); + if (i < 0) { + i += v.size(); + if (i < 0) { + throw index_error(); + } } - if ((SizeType) i >= v.size()) { + auto i_st = static_cast(i); + if (i_st >= v.size()) { throw index_error(); } - return v[(SizeType) i]; + return v[i_st]; }); cl.def( @@ -516,7 +525,7 @@ class_ bind_vector(handle scope, std::string const &name, A [](const Vector &v) -> bool { return !v.empty(); }, "Check whether the list is nonempty"); - cl.def("__len__", &Vector::size); + cl.def("__len__", [](const Vector &vec) { return vec.size(); }); #if 0 // C++ style functions deprecated, leaving it here as an example @@ -636,18 +645,52 @@ auto map_if_insertion_operator(Class_ &cl, std::string const &name) "Return the canonical string representation of this map."); } -template +template struct keys_view { - Map ↦ + virtual size_t len() = 0; + virtual iterator iter() = 0; + virtual bool contains(const KeyType &k) = 0; + virtual bool contains(const object &k) = 0; + virtual ~keys_view() = default; }; -template +template struct values_view { - Map ↦ + virtual size_t len() = 0; + virtual iterator iter() = 0; + virtual ~values_view() = default; }; -template +template struct items_view { + virtual size_t len() = 0; + virtual iterator iter() = 0; + virtual ~items_view() = default; +}; + +template +struct KeysViewImpl : public KeysView { + explicit KeysViewImpl(Map &map) : map(map) {} + size_t len() override { return map.size(); } + iterator iter() override { return make_key_iterator(map.begin(), map.end()); } + bool contains(const typename Map::key_type &k) override { return map.find(k) != map.end(); } + bool contains(const object &) override { return false; } + Map ↦ +}; + +template +struct ValuesViewImpl : public ValuesView { + explicit ValuesViewImpl(Map &map) : map(map) {} + size_t len() override { return map.size(); } + iterator iter() override { return make_value_iterator(map.begin(), map.end()); } + Map ↦ +}; + +template +struct ItemsViewImpl : public ItemsView { + explicit ItemsViewImpl(Map &map) : map(map) {} + size_t len() override { return map.size(); } + iterator iter() override { return make_iterator(map.begin(), map.end()); } Map ↦ }; @@ -657,9 +700,11 @@ template , typename... class_ bind_map(handle scope, const std::string &name, Args &&...args) { using KeyType = typename Map::key_type; using MappedType = typename Map::mapped_type; - using KeysView = detail::keys_view; - using ValuesView = detail::values_view; - using ItemsView = detail::items_view; + using StrippedKeyType = detail::remove_cvref_t; + using StrippedMappedType = detail::remove_cvref_t; + using KeysView = detail::keys_view; + using ValuesView = detail::values_view; + using ItemsView = detail::items_view; using Class_ = class_; // If either type is a non-module-local bound type then make the map binding non-local as well; @@ -673,12 +718,57 @@ class_ bind_map(handle scope, const std::string &name, Args && } Class_ cl(scope, name.c_str(), pybind11::module_local(local), std::forward(args)...); - class_ keys_view( - scope, ("KeysView[" + name + "]").c_str(), pybind11::module_local(local)); - class_ values_view( - scope, ("ValuesView[" + name + "]").c_str(), pybind11::module_local(local)); - class_ items_view( - scope, ("ItemsView[" + name + "]").c_str(), pybind11::module_local(local)); + static constexpr auto key_type_descr = detail::make_caster::name; + static constexpr auto mapped_type_descr = detail::make_caster::name; + std::string key_type_name(key_type_descr.text), mapped_type_name(mapped_type_descr.text); + + // If key type isn't properly wrapped, fall back to C++ names + if (key_type_name == "%") { + key_type_name = detail::type_info_description(typeid(KeyType)); + } + // Similarly for value type: + if (mapped_type_name == "%") { + mapped_type_name = detail::type_info_description(typeid(MappedType)); + } + + // Wrap KeysView[KeyType] if it wasn't already wrapped + if (!detail::get_type_info(typeid(KeysView))) { + class_ keys_view( + scope, ("KeysView[" + key_type_name + "]").c_str(), pybind11::module_local(local)); + keys_view.def("__len__", &KeysView::len); + keys_view.def("__iter__", + &KeysView::iter, + keep_alive<0, 1>() /* Essential: keep view alive while iterator exists */ + ); + keys_view.def("__contains__", + static_cast(&KeysView::contains)); + // Fallback for when the object is not of the key type + keys_view.def("__contains__", + static_cast(&KeysView::contains)); + } + // Similarly for ValuesView: + if (!detail::get_type_info(typeid(ValuesView))) { + class_ values_view(scope, + ("ValuesView[" + mapped_type_name + "]").c_str(), + pybind11::module_local(local)); + values_view.def("__len__", &ValuesView::len); + values_view.def("__iter__", + &ValuesView::iter, + keep_alive<0, 1>() /* Essential: keep view alive while iterator exists */ + ); + } + // Similarly for ItemsView: + if (!detail::get_type_info(typeid(ItemsView))) { + class_ items_view( + scope, + ("ItemsView[" + key_type_name + ", ").append(mapped_type_name + "]").c_str(), + pybind11::module_local(local)); + items_view.def("__len__", &ItemsView::len); + items_view.def("__iter__", + &ItemsView::iter, + keep_alive<0, 1>() /* Essential: keep view alive while iterator exists */ + ); + } cl.def(init<>()); @@ -698,19 +788,25 @@ class_ bind_map(handle scope, const std::string &name, Args && cl.def( "keys", - [](Map &m) { return KeysView{m}; }, + [](Map &m) { + return std::unique_ptr(new detail::KeysViewImpl(m)); + }, keep_alive<0, 1>() /* Essential: keep map alive while view exists */ ); cl.def( "values", - [](Map &m) { return ValuesView{m}; }, + [](Map &m) { + return std::unique_ptr(new detail::ValuesViewImpl(m)); + }, keep_alive<0, 1>() /* Essential: keep map alive while view exists */ ); cl.def( "items", - [](Map &m) { return ItemsView{m}; }, + [](Map &m) { + return std::unique_ptr(new detail::ItemsViewImpl(m)); + }, keep_alive<0, 1>() /* Essential: keep map alive while view exists */ ); @@ -747,37 +843,8 @@ class_ bind_map(handle scope, const std::string &name, Args && m.erase(it); }); - cl.def("__len__", &Map::size); - - keys_view.def("__len__", [](KeysView &view) { return view.map.size(); }); - keys_view.def( - "__iter__", - [](KeysView &view) { return make_key_iterator(view.map.begin(), view.map.end()); }, - keep_alive<0, 1>() /* Essential: keep view alive while iterator exists */ - ); - keys_view.def("__contains__", [](KeysView &view, const KeyType &k) -> bool { - auto it = view.map.find(k); - if (it == view.map.end()) { - return false; - } - return true; - }); - // Fallback for when the object is not of the key type - keys_view.def("__contains__", [](KeysView &, const object &) -> bool { return false; }); - - values_view.def("__len__", [](ValuesView &view) { return view.map.size(); }); - values_view.def( - "__iter__", - [](ValuesView &view) { return make_value_iterator(view.map.begin(), view.map.end()); }, - keep_alive<0, 1>() /* Essential: keep view alive while iterator exists */ - ); - - items_view.def("__len__", [](ItemsView &view) { return view.map.size(); }); - items_view.def( - "__iter__", - [](ItemsView &view) { return make_iterator(view.map.begin(), view.map.end()); }, - keep_alive<0, 1>() /* Essential: keep view alive while iterator exists */ - ); + // Always use a lambda in case of `using` declaration + cl.def("__len__", [](const Map &m) { return m.size(); }); return cl; } diff --git a/lib/pybind11/include/pybind11/type_caster_pyobject_ptr.h b/lib/pybind11/include/pybind11/type_caster_pyobject_ptr.h new file mode 100644 index 00000000..aa914f9e --- /dev/null +++ b/lib/pybind11/include/pybind11/type_caster_pyobject_ptr.h @@ -0,0 +1,61 @@ +// Copyright (c) 2023 The pybind Community. + +#pragma once + +#include "detail/common.h" +#include "detail/descr.h" +#include "cast.h" +#include "pytypes.h" + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(detail) + +template <> +class type_caster { +public: + static constexpr auto name = const_name("object"); // See discussion under PR #4601. + + // This overload is purely to guard against accidents. + template ::value, int> = 0> + static handle cast(T &&, return_value_policy, handle /*parent*/) { + static_assert(is_same_ignoring_cvref::value, + "Invalid C++ type T for to-Python conversion (type_caster)."); + return nullptr; // Unreachable. + } + + static handle cast(PyObject *src, return_value_policy policy, handle /*parent*/) { + if (src == nullptr) { + throw error_already_set(); + } + if (PyErr_Occurred()) { + raise_from(PyExc_SystemError, "src != nullptr but PyErr_Occurred()"); + throw error_already_set(); + } + if (policy == return_value_policy::take_ownership) { + return src; + } + if (policy == return_value_policy::reference + || policy == return_value_policy::automatic_reference) { + return handle(src).inc_ref(); + } + pybind11_fail("type_caster::cast(): unsupported return_value_policy: " + + std::to_string(static_cast(policy))); + } + + bool load(handle src, bool) { + value = reinterpret_borrow(src); + return true; + } + + template + using cast_op_type = PyObject *; + + explicit operator PyObject *() { return value.ptr(); } + +private: + object value; +}; + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/lib/pybind11/include/pybind11/typing.h b/lib/pybind11/include/pybind11/typing.h new file mode 100644 index 00000000..b7b1a4e5 --- /dev/null +++ b/lib/pybind11/include/pybind11/typing.h @@ -0,0 +1,117 @@ +/* + pybind11/typing.h: Convenience wrapper classes for basic Python types + with more explicit annotations. + + Copyright (c) 2023 Dustin Spicuzza + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#pragma once + +#include "detail/common.h" +#include "cast.h" +#include "pytypes.h" + +PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) +PYBIND11_NAMESPACE_BEGIN(typing) + +/* + The following types can be used to direct pybind11-generated docstrings + to have have more explicit types (e.g., `list[str]` instead of `list`). + Just use these in place of existing types. + + There is no additional enforcement of types at runtime. +*/ + +template +class Tuple : public tuple { + using tuple::tuple; +}; + +template +class Dict : public dict { + using dict::dict; +}; + +template +class List : public list { + using list::list; +}; + +template +class Set : public set { + using set::set; +}; + +template +class Iterable : public iterable { + using iterable::iterable; +}; + +template +class Iterator : public iterator { + using iterator::iterator; +}; + +template +class Callable; + +template +class Callable : public function { + using function::function; +}; + +PYBIND11_NAMESPACE_END(typing) + +PYBIND11_NAMESPACE_BEGIN(detail) + +template +struct handle_type_name> { + static constexpr auto name + = const_name("tuple[") + concat(make_caster::name...) + const_name("]"); +}; + +template <> +struct handle_type_name> { + // PEP 484 specifies this syntax for an empty tuple + static constexpr auto name = const_name("tuple[()]"); +}; + +template +struct handle_type_name> { + static constexpr auto name = const_name("dict[") + make_caster::name + const_name(", ") + + make_caster::name + const_name("]"); +}; + +template +struct handle_type_name> { + static constexpr auto name = const_name("list[") + make_caster::name + const_name("]"); +}; + +template +struct handle_type_name> { + static constexpr auto name = const_name("set[") + make_caster::name + const_name("]"); +}; + +template +struct handle_type_name> { + static constexpr auto name = const_name("Iterable[") + make_caster::name + const_name("]"); +}; + +template +struct handle_type_name> { + static constexpr auto name = const_name("Iterator[") + make_caster::name + const_name("]"); +}; + +template +struct handle_type_name> { + using retval_type = conditional_t::value, void_type, Return>; + static constexpr auto name = const_name("Callable[[") + concat(make_caster::name...) + + const_name("], ") + make_caster::name + + const_name("]"); +}; + +PYBIND11_NAMESPACE_END(detail) +PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/lib/pybind11/noxfile.py b/lib/pybind11/noxfile.py index 021ced24..f95b6491 100644 --- a/lib/pybind11/noxfile.py +++ b/lib/pybind11/noxfile.py @@ -57,7 +57,7 @@ def tests_packaging(session: nox.Session) -> None: Run the packaging tests. """ - session.install("-r", "tests/requirements.txt", "--prefer-binary") + session.install("-r", "tests/requirements.txt") session.run("pytest", "tests/extra_python_package", *session.posargs) diff --git a/lib/pybind11/pybind11/__init__.py b/lib/pybind11/pybind11/__init__.py index 4fbb1707..7c10b305 100644 --- a/lib/pybind11/pybind11/__init__.py +++ b/lib/pybind11/pybind11/__init__.py @@ -1,6 +1,6 @@ import sys -if sys.version_info < (3, 6): +if sys.version_info < (3, 6): # noqa: UP036 msg = "pybind11 does not support Python < 3.6. 2.9 was the last release supporting Python 2.7 and 3.5." raise ImportError(msg) diff --git a/lib/pybind11/pybind11/__main__.py b/lib/pybind11/pybind11/__main__.py index 8c895338..180665c2 100644 --- a/lib/pybind11/pybind11/__main__.py +++ b/lib/pybind11/pybind11/__main__.py @@ -4,6 +4,7 @@ import sys import sysconfig +from ._version import __version__ from .commands import get_cmake_dir, get_include, get_pkgconfig_dir @@ -24,8 +25,13 @@ def print_includes() -> None: def main() -> None: - parser = argparse.ArgumentParser() + parser.add_argument( + "--version", + action="version", + version=__version__, + help="Print the version and exit.", + ) parser.add_argument( "--includes", action="store_true", diff --git a/lib/pybind11/pybind11/_version.py b/lib/pybind11/pybind11/_version.py index 1cb51fc5..7934b7f7 100644 --- a/lib/pybind11/pybind11/_version.py +++ b/lib/pybind11/pybind11/_version.py @@ -8,5 +8,5 @@ def _to_int(s: str) -> Union[int, str]: return s -__version__ = "2.11.0.dev1" +__version__ = "2.12.0.dev1" version_info = tuple(_to_int(s) for s in __version__.split(".")) diff --git a/lib/pybind11/pybind11/commands.py b/lib/pybind11/pybind11/commands.py index 152fa20c..b11690f4 100644 --- a/lib/pybind11/pybind11/commands.py +++ b/lib/pybind11/pybind11/commands.py @@ -3,7 +3,7 @@ DIR = os.path.abspath(os.path.dirname(__file__)) -def get_include(user: bool = False) -> str: # pylint: disable=unused-argument +def get_include(user: bool = False) -> str: # noqa: ARG001 """ Return the path to the pybind11 include directory. The historical "user" argument is unused, and may be removed. diff --git a/lib/pybind11/pybind11/setup_helpers.py b/lib/pybind11/pybind11/setup_helpers.py index 1fd04b91..3b16dca8 100644 --- a/lib/pybind11/pybind11/setup_helpers.py +++ b/lib/pybind11/pybind11/setup_helpers.py @@ -66,8 +66,10 @@ from setuptools import Extension as _Extension from setuptools.command.build_ext import build_ext as _build_ext except ImportError: - from distutils.command.build_ext import build_ext as _build_ext - from distutils.extension import Extension as _Extension + from distutils.command.build_ext import ( # type: ignore[assignment] + build_ext as _build_ext, + ) + from distutils.extension import Extension as _Extension # type: ignore[assignment] import distutils.ccompiler import distutils.errors @@ -84,7 +86,7 @@ # directory into your path if it sits beside your setup.py. -class Pybind11Extension(_Extension): # type: ignore[misc] +class Pybind11Extension(_Extension): """ Build a C++11+ Extension module with pybind11. This automatically adds the recommended flags when you init the extension and assumes C++ sources - you @@ -118,7 +120,6 @@ def _add_ldflags(self, flags: List[str]) -> None: self.extra_link_args[:0] = flags def __init__(self, *args: Any, **kwargs: Any) -> None: - self._cxx_level = 0 cxx_std = kwargs.pop("cxx_std", 0) @@ -145,7 +146,6 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: self.cxx_std = cxx_std cflags = [] - ldflags = [] if WIN: cflags += ["/EHsc", "/bigobj"] else: @@ -155,11 +155,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: c_cpp_flags = shlex.split(env_cflags) + shlex.split(env_cppflags) if not any(opt.startswith("-g") for opt in c_cpp_flags): cflags += ["-g0"] - if MACOS: - cflags += ["-stdlib=libc++"] - ldflags += ["-stdlib=libc++"] self._add_cflags(cflags) - self._add_ldflags(ldflags) @property def cxx_std(self) -> int: @@ -174,9 +170,10 @@ def cxx_std(self) -> int: @cxx_std.setter def cxx_std(self, level: int) -> None: - if self._cxx_level: - warnings.warn("You cannot safely change the cxx_level after setting it!") + warnings.warn( + "You cannot safely change the cxx_level after setting it!", stacklevel=2 + ) # MSVC 2015 Update 3 and later only have 14 (and later 17) modes, so # force a valid flag here. @@ -271,7 +268,7 @@ def auto_cpp_level(compiler: Any) -> Union[str, int]: raise RuntimeError(msg) -class build_ext(_build_ext): # type: ignore[misc] # noqa: N801 +class build_ext(_build_ext): # noqa: N801 """ Customized build_ext that allows an auto-search for the highest supported C++ level for Pybind11Extension. This is only needed for the auto-search @@ -341,7 +338,7 @@ def naive_recompile(obj: str, src: str) -> bool: return os.stat(obj).st_mtime < os.stat(src).st_mtime -def no_recompile(obg: str, src: str) -> bool: # pylint: disable=unused-argument +def no_recompile(obg: str, src: str) -> bool: # noqa: ARG001 """ This is the safest but slowest choice (and is the default) - will always recompile sources. @@ -439,7 +436,6 @@ def compile_function( extra_postargs: Optional[List[str]] = None, depends: Optional[List[str]] = None, ) -> Any: - # These lines are directly from distutils.ccompiler.CCompiler macros, objects, extra_postargs, pp_opts, build = compiler._setup_compile( # type: ignore[attr-defined] output_dir, macros, include_dirs, sources, depends, extra_postargs diff --git a/lib/pybind11/pyproject.toml b/lib/pybind11/pyproject.toml index 3ba1b4b2..5af6015a 100644 --- a/lib/pybind11/pyproject.toml +++ b/lib/pybind11/pyproject.toml @@ -2,6 +2,7 @@ requires = ["setuptools>=42", "cmake>=3.18", "ninja"] build-backend = "setuptools.build_meta" + [tool.check-manifest] ignore = [ "tests/**", @@ -15,22 +16,17 @@ ignore = [ "noxfile.py", ] -[tool.isort] -# Needs the compiled .so modules and env.py from tests -known_first_party = "env,pybind11_cross_module_tests,pybind11_tests," -# For black compatibility -profile = "black" [tool.mypy] files = ["pybind11"] -python_version = "3.6" +python_version = "3.7" strict = true show_error_codes = true enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"] warn_unreachable = true [[tool.mypy.overrides]] -module = ["ghapi.*", "setuptools.*"] +module = ["ghapi.*"] ignore_missing_imports = true @@ -58,4 +54,42 @@ messages_control.disable = [ "invalid-name", "protected-access", "missing-module-docstring", + "unused-argument", # covered by Ruff ARG +] + +[tool.ruff] +target-version = "py37" +src = ["src"] + +[tool.ruff.lint] +extend-select = [ + "B", # flake8-bugbear + "I", # isort + "N", # pep8-naming + "ARG", # flake8-unused-arguments + "C4", # flake8-comprehensions + "EM", # flake8-errmsg + "ICN", # flake8-import-conventions + "PGH", # pygrep-hooks + "PIE", # flake8-pie + "PL", # pylint + "PT", # flake8-pytest-style + "RET", # flake8-return + "RUF100", # Ruff-specific + "SIM", # flake8-simplify + "UP", # pyupgrade + "YTT", # flake8-2020 ] +ignore = [ + "PLR", # Design related pylint + "E501", # Line too long (Black is enough) + "PT011", # Too broad with raises in pytest + "PT004", # Fixture that doesn't return needs underscore (no, it is fine) + "SIM118", # iter(x) is not always the same as iter(x.keys()) +] +unfixable = ["T20"] +isort.known-first-party = ["env", "pybind11_cross_module_tests", "pybind11_tests"] + +[tool.ruff.lint.per-file-ignores] +"tests/**" = ["EM", "N", "E721"] +"tests/test_call_policies.py" = ["PLC1901"] diff --git a/lib/pybind11/setup.cfg b/lib/pybind11/setup.cfg index 9af50ea4..92e6c953 100644 --- a/lib/pybind11/setup.cfg +++ b/lib/pybind11/setup.cfg @@ -20,6 +20,7 @@ classifiers = Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 License :: OSI Approved :: BSD License Programming Language :: Python :: Implementation :: PyPy Programming Language :: Python :: Implementation :: CPython @@ -40,11 +41,3 @@ project_urls = [options] python_requires = >=3.6 zip_safe = False - - -[flake8] -max-line-length = 120 -show_source = True -exclude = .git, __pycache__, build, dist, docs, tools, venv -extend-ignore = E203, E722, B903, B950 -extend-select = B9 diff --git a/lib/pybind11/setup.py b/lib/pybind11/setup.py index 68573519..9fea7d35 100644 --- a/lib/pybind11/setup.py +++ b/lib/pybind11/setup.py @@ -96,7 +96,7 @@ def get_and_replace( # Use our input files instead when making the SDist (and anything that depends # on it, like a wheel) -class SDist(setuptools.command.sdist.sdist): # type: ignore[misc] +class SDist(setuptools.command.sdist.sdist): def make_release_tree(self, base_dir: str, files: List[str]) -> None: super().make_release_tree(base_dir, files) diff --git a/lib/pybind11/tools/FindCatch.cmake b/lib/pybind11/tools/FindCatch.cmake index 57bba58b..5d3fcbfb 100644 --- a/lib/pybind11/tools/FindCatch.cmake +++ b/lib/pybind11/tools/FindCatch.cmake @@ -36,10 +36,14 @@ endfunction() function(_download_catch version destination_dir) message(STATUS "Downloading catch v${version}...") set(url https://github.com/philsquared/Catch/releases/download/v${version}/catch.hpp) - file(DOWNLOAD ${url} "${destination_dir}/catch.hpp" STATUS status) + file( + DOWNLOAD ${url} "${destination_dir}/catch.hpp" + STATUS status + LOG log) list(GET status 0 error) if(error) - message(FATAL_ERROR "Could not download ${url}") + string(REPLACE "\n" "\n " log " ${log}") + message(FATAL_ERROR "Could not download URL:\n" " ${url}\n" "Log:\n" "${log}") endif() set(CATCH_INCLUDE_DIR "${destination_dir}" diff --git a/lib/pybind11/tools/FindPythonLibsNew.cmake b/lib/pybind11/tools/FindPythonLibsNew.cmake index bbbd9f9e..8275b9d5 100644 --- a/lib/pybind11/tools/FindPythonLibsNew.cmake +++ b/lib/pybind11/tools/FindPythonLibsNew.cmake @@ -95,6 +95,22 @@ if(NOT PythonLibsNew_FIND_VERSION) set(PythonLibsNew_FIND_VERSION "3.6") endif() +if(NOT CMAKE_VERSION VERSION_LESS "3.27") + cmake_policy(GET CMP0148 _pybind11_cmp0148) + if(NOT _pybind11_cmp0148) + message( + AUTHOR_WARNING + "Policy CMP0148 is not set: The FindPythonInterp and FindPythonLibs " + "modules are removed. Run \"cmake --help-policy CMP0148\" for policy " + "details. Use the cmake_policy command to set the policy and suppress " + "this warning, or preferably upgrade to using FindPython, either by " + "calling it explicitly before pybind11, or by setting " + "PYBIND11_FINDPYTHON ON before pybind11.") + endif() + cmake_policy(SET CMP0148 OLD) + unset(_pybind11_cmp0148) +endif() + find_package(PythonInterp ${PythonLibsNew_FIND_VERSION} ${_pythonlibs_required} ${_pythonlibs_quiet}) @@ -172,13 +188,20 @@ _pybind11_get_if_undef(_PYTHON_VALUES 0 _PYTHON_VERSION_LIST) _pybind11_get_if_undef(_PYTHON_VALUES 1 PYTHON_PREFIX) _pybind11_get_if_undef(_PYTHON_VALUES 2 PYTHON_INCLUDE_DIR) _pybind11_get_if_undef(_PYTHON_VALUES 3 PYTHON_SITE_PACKAGES) -_pybind11_get_if_undef(_PYTHON_VALUES 4 PYTHON_MODULE_EXTENSION) _pybind11_get_if_undef(_PYTHON_VALUES 5 PYTHON_IS_DEBUG) _pybind11_get_if_undef(_PYTHON_VALUES 6 PYTHON_SIZEOF_VOID_P) _pybind11_get_if_undef(_PYTHON_VALUES 7 PYTHON_LIBRARY_SUFFIX) _pybind11_get_if_undef(_PYTHON_VALUES 8 PYTHON_LIBDIR) _pybind11_get_if_undef(_PYTHON_VALUES 9 PYTHON_MULTIARCH) +list(GET _PYTHON_VALUES 4 _PYTHON_MODULE_EXT_SUFFIX) +if(PYBIND11_PYTHONLIBS_OVERWRITE OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX) + get_filename_component(PYTHON_MODULE_DEBUG_POSTFIX "${_PYTHON_MODULE_EXT_SUFFIX}" NAME_WE) +endif() +if(PYBIND11_PYTHONLIBS_OVERWRITE OR NOT DEFINED PYTHON_MODULE_EXTENSION) + get_filename_component(PYTHON_MODULE_EXTENSION "${_PYTHON_MODULE_EXT_SUFFIX}" EXT) +endif() + # Make sure the Python has the same pointer-size as the chosen compiler # Skip if CMAKE_SIZEOF_VOID_P is not defined # This should be skipped for (non-Apple) cross-compiles (like EMSCRIPTEN) @@ -208,7 +231,9 @@ string(REGEX REPLACE "\\\\" "/" PYTHON_PREFIX "${PYTHON_PREFIX}") string(REGEX REPLACE "\\\\" "/" PYTHON_INCLUDE_DIR "${PYTHON_INCLUDE_DIR}") string(REGEX REPLACE "\\\\" "/" PYTHON_SITE_PACKAGES "${PYTHON_SITE_PACKAGES}") -if(CMAKE_HOST_WIN32) +if(DEFINED PYTHON_LIBRARY) + # Don't write to PYTHON_LIBRARY if it's already set +elseif(CMAKE_HOST_WIN32) set(PYTHON_LIBRARY "${PYTHON_PREFIX}/libs/python${PYTHON_LIBRARY_SUFFIX}.lib") # when run in a venv, PYTHON_PREFIX points to it. But the libraries remain in the @@ -274,7 +299,7 @@ if(NOT PYTHON_DEBUG_LIBRARY) endif() set(PYTHON_DEBUG_LIBRARIES "${PYTHON_DEBUG_LIBRARY}") -find_package_message(PYTHON "Found PythonLibs: ${PYTHON_LIBRARY}" +find_package_message(PYTHON "Found PythonLibs: ${PYTHON_LIBRARIES}" "${PYTHON_EXECUTABLE}${PYTHON_VERSION_STRING}") set(PYTHONLIBS_FOUND TRUE) diff --git a/lib/pybind11/tools/codespell_ignore_lines_from_errors.py b/lib/pybind11/tools/codespell_ignore_lines_from_errors.py index 5403ec3a..4ec9add1 100644 --- a/lib/pybind11/tools/codespell_ignore_lines_from_errors.py +++ b/lib/pybind11/tools/codespell_ignore_lines_from_errors.py @@ -17,14 +17,18 @@ def run(args: List[str]) -> None: assert len(args) == 1, "codespell_errors.txt" cache = {} done = set() - for line in sorted(open(args[0]).read().splitlines()): + with open(args[0]) as f: + lines = f.read().splitlines() + + for line in sorted(lines): i = line.find(" ==> ") if i > 0: flds = line[:i].split(":") if len(flds) >= 2: filename, line_num = flds[:2] if filename not in cache: - cache[filename] = open(filename).read().splitlines() + with open(filename) as f: + cache[filename] = f.read().splitlines() supp = cache[filename][int(line_num) - 1] if supp not in done: print(supp) diff --git a/lib/pybind11/tools/make_changelog.py b/lib/pybind11/tools/make_changelog.py index 839040a9..3cdf47e8 100755 --- a/lib/pybind11/tools/make_changelog.py +++ b/lib/pybind11/tools/make_changelog.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +from __future__ import annotations import re @@ -29,24 +30,48 @@ ) issues = (issue for page in issues_pages for issue in page) missing = [] +cats_descr = { + "feat": "New Features", + "fix": "Bug fixes", + "fix(types)": "", + "fix(cmake)": "", + "docs": "Documentation", + "tests": "Tests", + "ci": "CI", + "chore": "Other", + "unknown": "Uncategorised", +} +cats: dict[str, list[str]] = {c: [] for c in cats_descr} for issue in issues: - changelog = ENTRY.findall(issue.body) - if changelog: + changelog = ENTRY.findall(issue.body or "") + if not changelog or not changelog[0]: + missing.append(issue) + else: (msg,) = changelog + if msg.startswith("- "): + msg = msg[2:] if not msg.startswith("* "): msg = "* " + msg if not msg.endswith("."): msg += "." msg += f"\n `#{issue.number} <{issue.html_url}>`_" + for cat in cats: + if issue.title.lower().startswith(f"{cat}:"): + cats[cat].append(msg) + break + else: + cats["unknown"].append(msg) - print(Syntax(msg, "rst", theme="ansi_light", word_wrap=True)) +for cat, msgs in cats.items(): + if msgs: + desc = cats_descr[cat] + print(f"[bold]{desc}:\n" if desc else "") + for msg in msgs: + print(Syntax(msg, "rst", theme="ansi_light", word_wrap=True)) print() - else: - missing.append(issue) - if missing: print() print("[blue]" + "-" * 30) diff --git a/lib/pybind11/tools/pybind11Common.cmake b/lib/pybind11/tools/pybind11Common.cmake index e1fb601a..50b15e11 100644 --- a/lib/pybind11/tools/pybind11Common.cmake +++ b/lib/pybind11/tools/pybind11Common.cmake @@ -5,8 +5,8 @@ Adds the following targets:: pybind11::pybind11 - link to headers and pybind11 pybind11::module - Adds module links pybind11::embed - Adds embed links - pybind11::lto - Link time optimizations (manual selection) - pybind11::thin_lto - Link time optimizations (manual selection) + pybind11::lto - Link time optimizations (only if CMAKE_INTERPROCEDURAL_OPTIMIZATION is not set) + pybind11::thin_lto - Link time optimizations (only if CMAKE_INTERPROCEDURAL_OPTIMIZATION is not set) pybind11::python_link_helper - Adds link to Python libraries pybind11::windows_extras - MSVC bigobj and mp for building multithreaded pybind11::opt_size - avoid optimizations that increase code size @@ -20,7 +20,7 @@ Adds the following functions:: # CMake 3.10 has an include_guard command, but we can't use that yet # include_guard(global) (pre-CMake 3.10) -if(TARGET pybind11::lto) +if(TARGET pybind11::pybind11) return() endif() @@ -163,11 +163,19 @@ endif() # --------------------- Python specifics ------------------------- +# CMake 3.27 removes the classic FindPythonInterp if CMP0148 is NEW +if(CMAKE_VERSION VERSION_LESS "3.27") + set(_pybind11_missing_old_python "OLD") +else() + cmake_policy(GET CMP0148 _pybind11_missing_old_python) +endif() + # Check to see which Python mode we are in, new, old, or no python if(PYBIND11_NOPYTHON) set(_pybind11_nopython ON) elseif( - PYBIND11_FINDPYTHON + _pybind11_missing_old_python STREQUAL "NEW" + OR PYBIND11_FINDPYTHON OR Python_FOUND OR Python2_FOUND OR Python3_FOUND) @@ -210,8 +218,15 @@ if(NOT _pybind11_nopython) execute_process( COMMAND - ${${_Python}_EXECUTABLE} -c - "from pkg_resources import get_distribution; print(get_distribution('${PYPI_NAME}').version)" + ${${_Python}_EXECUTABLE} -c " +try: + from importlib.metadata import version +except ImportError: + from pkg_resources import get_distribution + def version(s): + return get_distribution(s).version +print(version('${PYPI_NAME}')) + " RESULT_VARIABLE RESULT_PRESENT OUTPUT_VARIABLE PKG_VERSION ERROR_QUIET) @@ -292,25 +307,38 @@ function(_pybind11_generate_lto target prefer_thin_lto) set(cxx_append ";-fno-fat-lto-objects") endif() - if(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le" OR CMAKE_SYSTEM_PROCESSOR MATCHES "mips64") - set(NO_FLTO_ARCH TRUE) + if(prefer_thin_lto) + set(thin "=thin") else() - set(NO_FLTO_ARCH FALSE) + set(thin "") endif() - if(CMAKE_CXX_COMPILER_ID MATCHES "Clang" - AND prefer_thin_lto - AND NOT NO_FLTO_ARCH) + if(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le" OR CMAKE_SYSTEM_PROCESSOR MATCHES "mips64") + # Do nothing + elseif(CMAKE_SYSTEM_PROCESSOR MATCHES emscripten) + # This compile is very costly when cross-compiling, so set this without checking + set(PYBIND11_LTO_CXX_FLAGS "-flto${thin}${cxx_append}") + set(PYBIND11_LTO_LINKER_FLAGS "-flto${thin}${linker_append}") + elseif(CMAKE_CXX_COMPILER_ID MATCHES "Clang") _pybind11_return_if_cxx_and_linker_flags_work( - HAS_FLTO_THIN "-flto=thin${cxx_append}" "-flto=thin${linker_append}" + HAS_FLTO_THIN "-flto${thin}${cxx_append}" "-flto=${thin}${linker_append}" PYBIND11_LTO_CXX_FLAGS PYBIND11_LTO_LINKER_FLAGS) endif() - - if(NOT HAS_FLTO_THIN AND NOT NO_FLTO_ARCH) + if(NOT HAS_FLTO_THIN) _pybind11_return_if_cxx_and_linker_flags_work( HAS_FLTO "-flto${cxx_append}" "-flto${linker_append}" PYBIND11_LTO_CXX_FLAGS PYBIND11_LTO_LINKER_FLAGS) endif() + elseif(CMAKE_CXX_COMPILER_ID MATCHES "IntelLLVM") + # IntelLLVM equivalent to LTO is called IPO; also IntelLLVM is WIN32/UNIX + # WARNING/HELP WANTED: This block of code is currently not covered by pybind11 GitHub Actions! + if(WIN32) + _pybind11_return_if_cxx_and_linker_flags_work( + HAS_INTEL_IPO "-Qipo" "-Qipo" PYBIND11_LTO_CXX_FLAGS PYBIND11_LTO_LINKER_FLAGS) + else() + _pybind11_return_if_cxx_and_linker_flags_work( + HAS_INTEL_IPO "-ipo" "-ipo" PYBIND11_LTO_CXX_FLAGS PYBIND11_LTO_LINKER_FLAGS) + endif() elseif(CMAKE_CXX_COMPILER_ID MATCHES "Intel") # Intel equivalent to LTO is called IPO _pybind11_return_if_cxx_and_linker_flags_work(HAS_INTEL_IPO "-ipo" "-ipo" @@ -362,11 +390,13 @@ function(_pybind11_generate_lto target prefer_thin_lto) endif() endfunction() -add_library(pybind11::lto IMPORTED INTERFACE ${optional_global}) -_pybind11_generate_lto(pybind11::lto FALSE) +if(NOT DEFINED CMAKE_INTERPROCEDURAL_OPTIMIZATION) + add_library(pybind11::lto IMPORTED INTERFACE ${optional_global}) + _pybind11_generate_lto(pybind11::lto FALSE) -add_library(pybind11::thin_lto IMPORTED INTERFACE ${optional_global}) -_pybind11_generate_lto(pybind11::thin_lto TRUE) + add_library(pybind11::thin_lto IMPORTED INTERFACE ${optional_global}) + _pybind11_generate_lto(pybind11::thin_lto TRUE) +endif() # ---------------------- pybind11_strip ----------------------------- diff --git a/lib/pybind11/tools/pybind11Config.cmake.in b/lib/pybind11/tools/pybind11Config.cmake.in index 9383e8c6..304f1d90 100644 --- a/lib/pybind11/tools/pybind11Config.cmake.in +++ b/lib/pybind11/tools/pybind11Config.cmake.in @@ -63,7 +63,9 @@ Modes There are two modes provided; classic, which is built on the old Python discovery packages in CMake, or the new FindPython mode, which uses FindPython -from 3.12+ forward (3.15+ _highly_ recommended). +from 3.12+ forward (3.15+ _highly_ recommended). If you set the minimum or +maximum version of CMake to 3.27+, then FindPython is the default (since +FindPythonInterp/FindPythonLibs has been removed via policy `CMP0148`). New FindPython mode ^^^^^^^^^^^^^^^^^^^ @@ -147,7 +149,7 @@ default is ``MODULE``. There are several options: ``OPT_SIZE`` Optimize for size, even if the ``CMAKE_BUILD_TYPE`` is not ``MinSizeRel``. ``THIN_LTO`` - Use thin TLO instead of regular if there's a choice (pybind11's selection + Use thin LTO instead of regular if there's a choice (pybind11's selection is disabled if ``CMAKE_INTERPROCEDURAL_OPTIMIZATIONS`` is set). ``WITHOUT_SOABI`` Disable the SOABI component (``PYBIND11_NEWPYTHON`` mode only). diff --git a/lib/pybind11/tools/pybind11NewTools.cmake b/lib/pybind11/tools/pybind11NewTools.cmake index 5a6a0cb8..9fe2eb08 100644 --- a/lib/pybind11/tools/pybind11NewTools.cmake +++ b/lib/pybind11/tools/pybind11NewTools.cmake @@ -9,7 +9,7 @@ if(CMAKE_VERSION VERSION_LESS 3.12) message(FATAL_ERROR "You cannot use the new FindPython module with CMake < 3.12") endif() -include_guard(GLOBAL) +include_guard(DIRECTORY) get_property( is_config @@ -32,17 +32,54 @@ if(NOT Python_FOUND AND NOT Python3_FOUND) set(Python_ROOT_DIR "$ENV{pythonLocation}") endif() - find_package(Python 3.6 REQUIRED COMPONENTS Interpreter Development ${_pybind11_quiet}) + # Development.Module support (required for manylinux) started in 3.18 + if(CMAKE_VERSION VERSION_LESS 3.18) + set(_pybind11_dev_component Development) + else() + set(_pybind11_dev_component Development.Module OPTIONAL_COMPONENTS Development.Embed) + endif() + + # Callers need to be able to access Python_EXECUTABLE + set(_pybind11_global_keyword "") + if(NOT is_config AND NOT DEFINED Python_ARTIFACTS_INTERACTIVE) + set(Python_ARTIFACTS_INTERACTIVE TRUE) + if(NOT CMAKE_VERSION VERSION_LESS 3.24) + set(_pybind11_global_keyword "GLOBAL") + endif() + endif() + + find_package(Python 3.6 REQUIRED COMPONENTS Interpreter ${_pybind11_dev_component} + ${_pybind11_quiet} ${_pybind11_global_keyword}) # If we are in submodule mode, export the Python targets to global targets. # If this behavior is not desired, FindPython _before_ pybind11. - if(NOT is_config) - set_property(TARGET Python::Python PROPERTY IMPORTED_GLOBAL TRUE) + if(NOT is_config + AND Python_ARTIFACTS_INTERACTIVE + AND _pybind11_global_keyword STREQUAL "") + if(TARGET Python::Python) + set_property(TARGET Python::Python PROPERTY IMPORTED_GLOBAL TRUE) + endif() set_property(TARGET Python::Interpreter PROPERTY IMPORTED_GLOBAL TRUE) if(TARGET Python::Module) set_property(TARGET Python::Module PROPERTY IMPORTED_GLOBAL TRUE) endif() endif() + + # Explicitly export version for callers (including our own functions) + if(NOT is_config AND Python_ARTIFACTS_INTERACTIVE) + set(Python_VERSION + "${Python_VERSION}" + CACHE INTERNAL "") + set(Python_VERSION_MAJOR + "${Python_VERSION_MAJOR}" + CACHE INTERNAL "") + set(Python_VERSION_MINOR + "${Python_VERSION_MINOR}" + CACHE INTERNAL "") + set(Python_VERSION_PATCH + "${Python_VERSION_PATCH}" + CACHE INTERNAL "") + endif() endif() if(Python_FOUND) @@ -73,15 +110,17 @@ if(NOT DEFINED ${_Python}_EXECUTABLE) endif() -if(NOT ${_Python}_EXECUTABLE STREQUAL PYBIND11_PYTHON_EXECUTABLE_LAST) +if(DEFINED PYBIND11_PYTHON_EXECUTABLE_LAST AND NOT ${_Python}_EXECUTABLE STREQUAL + PYBIND11_PYTHON_EXECUTABLE_LAST) # Detect changes to the Python version/binary in subsequent CMake runs, and refresh config if needed unset(PYTHON_IS_DEBUG CACHE) unset(PYTHON_MODULE_EXTENSION CACHE) - set(PYBIND11_PYTHON_EXECUTABLE_LAST - "${${_Python}_EXECUTABLE}" - CACHE INTERNAL "Python executable during the last CMake run") endif() +set(PYBIND11_PYTHON_EXECUTABLE_LAST + "${${_Python}_EXECUTABLE}" + CACHE INTERNAL "Python executable during the last CMake run") + if(NOT DEFINED PYTHON_IS_DEBUG) # Debug check - see https://stackoverflow.com/questions/646518/python-how-to-detect-debug-Interpreter execute_process( @@ -95,25 +134,36 @@ endif() # Get the suffix - SO is deprecated, should use EXT_SUFFIX, but this is # required for PyPy3 (as of 7.3.1) -if(NOT DEFINED PYTHON_MODULE_EXTENSION) +if(NOT DEFINED PYTHON_MODULE_EXTENSION OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX) execute_process( COMMAND "${${_Python}_EXECUTABLE}" "-c" "import sys, importlib; s = importlib.import_module('distutils.sysconfig' if sys.version_info < (3, 10) else 'sysconfig'); print(s.get_config_var('EXT_SUFFIX') or s.get_config_var('SO'))" - OUTPUT_VARIABLE _PYTHON_MODULE_EXTENSION - ERROR_VARIABLE _PYTHON_MODULE_EXTENSION_ERR + OUTPUT_VARIABLE _PYTHON_MODULE_EXT_SUFFIX + ERROR_VARIABLE _PYTHON_MODULE_EXT_SUFFIX_ERR OUTPUT_STRIP_TRAILING_WHITESPACE) - if(_PYTHON_MODULE_EXTENSION STREQUAL "") + if(_PYTHON_MODULE_EXT_SUFFIX STREQUAL "") message( FATAL_ERROR "pybind11 could not query the module file extension, likely the 'distutils'" - "package is not installed. Full error message:\n${_PYTHON_MODULE_EXTENSION_ERR}") + "package is not installed. Full error message:\n${_PYTHON_MODULE_EXT_SUFFIX_ERR}" + ) endif() # This needs to be available for the pybind11_extension function - set(PYTHON_MODULE_EXTENSION - "${_PYTHON_MODULE_EXTENSION}" - CACHE INTERNAL "") + if(NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX) + get_filename_component(_PYTHON_MODULE_DEBUG_POSTFIX "${_PYTHON_MODULE_EXT_SUFFIX}" NAME_WE) + set(PYTHON_MODULE_DEBUG_POSTFIX + "${_PYTHON_MODULE_DEBUG_POSTFIX}" + CACHE INTERNAL "") + endif() + + if(NOT DEFINED PYTHON_MODULE_EXTENSION) + get_filename_component(_PYTHON_MODULE_EXTENSION "${_PYTHON_MODULE_EXT_SUFFIX}" EXT) + set(PYTHON_MODULE_EXTENSION + "${_PYTHON_MODULE_EXTENSION}" + CACHE INTERNAL "") + endif() endif() # Python debug libraries expose slightly different objects before 3.8 @@ -233,11 +283,13 @@ function(pybind11_add_module target_name) endif() endif() - # Use case-insensitive comparison to match the result of $ - string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE) - if(NOT MSVC AND NOT "${uppercase_CMAKE_BUILD_TYPE}" MATCHES DEBUG|RELWITHDEBINFO) - # Strip unnecessary sections of the binary on Linux/macOS - pybind11_strip(${target_name}) + if(DEFINED CMAKE_BUILD_TYPE) # see https://github.com/pybind/pybind11/issues/4454 + # Use case-insensitive comparison to match the result of $ + string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE) + if(NOT MSVC AND NOT "${uppercase_CMAKE_BUILD_TYPE}" MATCHES DEBUG|RELWITHDEBINFO) + # Strip unnecessary sections of the binary on Linux/macOS + pybind11_strip(${target_name}) + endif() endif() if(MSVC) @@ -251,6 +303,9 @@ endfunction() function(pybind11_extension name) # The extension is precomputed - set_target_properties(${name} PROPERTIES PREFIX "" SUFFIX "${PYTHON_MODULE_EXTENSION}") - + set_target_properties( + ${name} + PROPERTIES PREFIX "" + DEBUG_POSTFIX "${PYTHON_MODULE_DEBUG_POSTFIX}" + SUFFIX "${PYTHON_MODULE_EXTENSION}") endfunction() diff --git a/lib/pybind11/tools/pybind11Tools.cmake b/lib/pybind11/tools/pybind11Tools.cmake index 66ad00a4..045e5f1e 100644 --- a/lib/pybind11/tools/pybind11Tools.cmake +++ b/lib/pybind11/tools/pybind11Tools.cmake @@ -43,7 +43,7 @@ endif() # A user can set versions manually too set(Python_ADDITIONAL_VERSIONS - "3.11;3.10;3.9;3.8;3.7;3.6" + "3.12;3.11;3.10;3.9;3.8;3.7;3.6" CACHE INTERNAL "") list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}") @@ -65,6 +65,7 @@ _pybind11_promote_to_cache(PYTHON_INCLUDE_DIRS) _pybind11_promote_to_cache(PYTHON_LIBRARIES) _pybind11_promote_to_cache(PYTHON_MODULE_PREFIX) _pybind11_promote_to_cache(PYTHON_MODULE_EXTENSION) +_pybind11_promote_to_cache(PYTHON_MODULE_DEBUG_POSTFIX) _pybind11_promote_to_cache(PYTHON_VERSION_MAJOR) _pybind11_promote_to_cache(PYTHON_VERSION_MINOR) _pybind11_promote_to_cache(PYTHON_VERSION) @@ -148,8 +149,11 @@ endif() function(pybind11_extension name) # The prefix and extension are provided by FindPythonLibsNew.cmake - set_target_properties(${name} PROPERTIES PREFIX "${PYTHON_MODULE_PREFIX}" - SUFFIX "${PYTHON_MODULE_EXTENSION}") + set_target_properties( + ${name} + PROPERTIES PREFIX "${PYTHON_MODULE_PREFIX}" + DEBUG_POSTFIX "${PYTHON_MODULE_DEBUG_POSTFIX}" + SUFFIX "${PYTHON_MODULE_EXTENSION}") endfunction() # Build a Python extension module: @@ -212,10 +216,12 @@ function(pybind11_add_module target_name) endif() endif() - # Use case-insensitive comparison to match the result of $ - string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE) - if(NOT MSVC AND NOT "${uppercase_CMAKE_BUILD_TYPE}" MATCHES DEBUG|RELWITHDEBINFO) - pybind11_strip(${target_name}) + if(DEFINED CMAKE_BUILD_TYPE) # see https://github.com/pybind/pybind11/issues/4454 + # Use case-insensitive comparison to match the result of $ + string(TOUPPER "${CMAKE_BUILD_TYPE}" uppercase_CMAKE_BUILD_TYPE) + if(NOT MSVC AND NOT "${uppercase_CMAKE_BUILD_TYPE}" MATCHES DEBUG|RELWITHDEBINFO) + pybind11_strip(${target_name}) + endif() endif() if(MSVC)