diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index 0248082..0000000 --- a/.appveyor.yml +++ /dev/null @@ -1,36 +0,0 @@ -build: false - -os: Visual Studio 2015 - -platform: - - x64 - - x86 - -environment: - matrix: - - MINICONDA: C:\xtensor-conda - -init: - - "ECHO %MINICONDA%" - - C:\"Program Files (x86)"\"Microsoft Visual Studio 14.0"\VC\vcvarsall.bat %PLATFORM% - - ps: if($env:Platform -eq "x64"){Start-FileDownload 'http://repo.continuum.io/miniconda/Miniconda3-latest-Windows-x86_64.exe' C:\Miniconda.exe; echo "Done"} - - ps: if($env:Platform -eq "x86"){Start-FileDownload 'http://repo.continuum.io/miniconda/Miniconda3-latest-Windows-x86.exe' C:\Miniconda.exe; echo "Done"} - - cmd: C:\Miniconda.exe /S /D=C:\xtensor-conda - - "set PATH=%MINICONDA%;%MINICONDA%\\Scripts;%MINICONDA%\\Library\\bin;%PATH%" - -install: - - conda config --set always_yes yes --set changeps1 no - - conda update -q conda - - conda info -a - - conda install gtest cmake -c conda-forge - - conda install pytest numpy pybind11==2.2.1 -c conda-forge - - conda install xtensor==0.15.1 -c QuantStack - - "set PYTHONHOME=%MINICONDA%" - - cmake -G "NMake Makefiles" -D CMAKE_INSTALL_PREFIX=%MINICONDA%\\Library -D BUILD_TESTS=ON -D PYTHON_EXECUTABLE=%MINICONDA%\\python.exe . - - nmake test_xtensor_python - - nmake install - -build_script: - - py.test -s - - cd test - - .\test_xtensor_python diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml new file mode 100644 index 0000000..ea4dee7 --- /dev/null +++ b/.github/workflows/linux.yml @@ -0,0 +1,104 @@ +name: Linux +on: + workflow_dispatch: + pull_request: + push: + branches: [master] +concurrency: + group: ${{ github.workflow }}-${{ github.job }}-${{ github.ref }} + cancel-in-progress: true +defaults: + run: + shell: bash -e -l {0} +jobs: + build: + runs-on: ubuntu-24.04 + name: ${{ matrix.sys.compiler }} ${{ matrix.sys.version }} + strategy: + fail-fast: false + matrix: + sys: + - {compiler: gcc, version: '11'} + - {compiler: gcc, version: '12'} + - {compiler: gcc, version: '13'} + - {compiler: gcc, version: '14'} + - {compiler: clang, version: '17'} + - {compiler: clang, version: '18'} + - {compiler: clang, version: '19'} + - {compiler: clang, version: '20'} + + steps: + - name: Install GCC + if: matrix.sys.compiler == 'gcc' + uses: egor-tensin/setup-gcc@v1 + with: + version: ${{matrix.sys.version}} + platform: x64 + + - name: Install LLVM and Clang + if: matrix.sys.compiler == 'clang' + run: | + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh ${{matrix.sys.version}} + sudo apt-get install -y clang-tools-${{matrix.sys.version}} + sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-${{matrix.sys.version}} 200 + sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-${{matrix.sys.version}} 200 + sudo update-alternatives --install /usr/bin/clang-scan-deps clang-scan-deps /usr/bin/clang-scan-deps-${{matrix.sys.version}} 200 + sudo update-alternatives --set clang /usr/bin/clang-${{matrix.sys.version}} + sudo update-alternatives --set clang++ /usr/bin/clang++-${{matrix.sys.version}} + sudo update-alternatives --set clang-scan-deps /usr/bin/clang-scan-deps-${{matrix.sys.version}} + + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set conda environment + uses: mamba-org/setup-micromamba@v1 + with: + environment-file: environment-dev.yml + cache-environment: true + + - name: Configure using CMake + run: cmake -G Ninja -Bbuild -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX -DPYTHON_EXECUTABLE=`which python` -DDOWNLOAD_GTEST=ON $(Build.SourcesDirectory) + + - name: Install + working-directory: build + run: cmake --install . + + - name: Build + working-directory: build + run: cmake --build . --target test_xtensor_python --parallel 8 + + - name: Run tests (C++) + working-directory: build/test + run: ./test_xtensor_python + + - name: Run tests (Python) + run: pytest -s + + - name: Example - readme 1 + working-directory: docs/source/examples/readme_example_1 + run: | + cmake -Bbuild -DPython_EXECUTABLE=`which python` + cd build + cmake --build . + cp ../example.py . + python example.py + + - name: Example - copy \'cast\' + working-directory: docs/source/examples/copy_cast + run: | + cmake -Bbuild -DPython_EXECUTABLE=`which python` + cd build + cmake --build . + cp ../example.py . + python example.py + + - name: Example - SFINAE + working-directory: docs/source/examples/sfinae + run: | + cmake -Bbuild -DPython_EXECUTABLE=`which python` + cd build + cmake --build . + cp ../example.py . + python example.py diff --git a/.github/workflows/osx.yml b/.github/workflows/osx.yml new file mode 100644 index 0000000..118d4fd --- /dev/null +++ b/.github/workflows/osx.yml @@ -0,0 +1,79 @@ +name: OSX +on: + workflow_dispatch: + pull_request: + push: + branches: [master] +concurrency: + group: ${{ github.workflow }}-${{ github.job }}-${{ github.ref }} + cancel-in-progress: true +defaults: + run: + shell: bash -e -l {0} +jobs: + build: + runs-on: macos-${{ matrix.os }} + name: macos-${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - 13 + - 14 + - 15 + + steps: + + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set conda environment + uses: mamba-org/setup-micromamba@v1 + with: + environment-file: environment-dev.yml + cache-environment: true + + - name: Configure using CMake + run: cmake -G Ninja -Bbuild -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX -DPYTHON_EXECUTABLE=`which python` -DDOWNLOAD_GTEST=ON $(Build.SourcesDirectory) + + - name: Install + working-directory: build + run: cmake --install . + + - name: Build + working-directory: build + run: cmake --build . --target test_xtensor_python --parallel 8 + + - name: Run tests (C++) + working-directory: build/test + run: ./test_xtensor_python + + - name: Run tests (Python) + run: pytest -s + + - name: Example - readme 1 + working-directory: docs/source/examples/readme_example_1 + run: | + cmake -Bbuild -DPython_EXECUTABLE=`which python` + cd build + cmake --build . + cp ../example.py . + python example.py + + - name: Example - copy \'cast\' + working-directory: docs/source/examples/copy_cast + run: | + cmake -Bbuild -DPython_EXECUTABLE=`which python` + cd build + cmake --build . + cp ../example.py . + python example.py + + - name: Example - SFINAE + working-directory: docs/source/examples/sfinae + run: | + cmake -Bbuild -DPython_EXECUTABLE=`which python` + cd build + cmake --build . + cp ../example.py . + python example.py diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml new file mode 100644 index 0000000..6743937 --- /dev/null +++ b/.github/workflows/windows.yml @@ -0,0 +1,48 @@ +name: Windows +on: + workflow_dispatch: + pull_request: + push: + branches: [master] +concurrency: + group: ${{ github.workflow }}-${{ github.job }}-${{ github.ref }} + cancel-in-progress: true +defaults: + run: + shell: bash -e -l {0} +jobs: + build: + runs-on: [windows-latest] + name: Windows + + steps: + + - name: Setup MSVC + uses: ilammy/msvc-dev-cmd@v1 + + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set conda environment + uses: mamba-org/setup-micromamba@v1 + with: + environment-file: environment-dev.yml + cache-environment: true + + - name: Configure using CMake + run: cmake -G Ninja -Bbuild -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX -DPYTHON_EXECUTABLE=`which python` -DDOWNLOAD_GTEST=ON $(Build.SourcesDirectory) + + - name: Install + working-directory: build + run: cmake --install . + + - name: Build + working-directory: build + run: cmake --build . --target test_xtensor_python --parallel 8 + + - name: Run tests (C++) + working-directory: build/test + run: ./test_xtensor_python + + - name: Run tests (Python) + run: pytest -s diff --git a/.gitignore b/.gitignore index 3d4abde..9413a44 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,7 @@ test/CMakeCache.txt test/Makefile test/CMakeFiles/ test/cmake_install.cmake +.pytest_cache/ # Documentation build artefacts docs/CMakeCache.txt diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 7057222..0000000 --- a/.travis.yml +++ /dev/null @@ -1,109 +0,0 @@ -language: cpp -dist: precise -matrix: - include: - - os: linux - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - g++-4.9 - env: COMPILER=gcc GCC=4.9 PY=3 - - os: linux - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - g++-5 - env: COMPILER=gcc GCC=5 PY=3 - - os: linux - addons: - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - g++-5 - env: COMPILER=gcc GCC=5 PY=2 - - os: linux - addons: - apt: - sources: - - ubuntu-toolchain-r-test - - llvm-toolchain-precise-3.6 - packages: - - clang-3.6 - env: COMPILER=clang CLANG=3.6 PY=3 - - os: linux - addons: - apt: - sources: - - ubuntu-toolchain-r-test - - llvm-toolchain-precise-3.7 - packages: - - clang-3.7 - env: COMPILER=clang CLANG=3.7 PY=3 - - os: linux - addons: - apt: - sources: - - ubuntu-toolchain-r-test - - llvm-toolchain-precise-3.8 - packages: - - clang-3.8 - env: COMPILER=clang CLANG=3.8 PY=3 - - os: osx - osx_image: xcode8 - compiler: clang - env: PY=3 -env: - global: - - MINCONDA_VERSION="4.3.21" - - MINCONDA_LINUX="Linux-x86_64" - - MINCONDA_OSX="MacOSX-x86_64" -before_install: - - | - # Configure build variables - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then - if [[ "$COMPILER" == "gcc" ]]; then - export CXX=g++-$GCC CC=gcc-$GCC; - fi - if [[ "$COMPILER" == "clang" ]]; then - export CXX=clang++-$CLANG CC=clang-$CLANG; - fi - elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then - export CXX=clang++ CC=clang PYTHONHOME=$HOME/miniconda; - fi - -install: - # Define the version of miniconda to download - - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then - MINCONDA_OS=$MINCONDA_LINUX; - elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then - MINCONDA_OS=$MINCONDA_OSX; - fi - - if [[ "$PY" == "3" ]]; then - wget "http://repo.continuum.io/miniconda/Miniconda3-$MINCONDA_VERSION-$MINCONDA_OS.sh" -O miniconda.sh; - else - wget "http://repo.continuum.io/miniconda/Miniconda2-$MINCONDA_VERSION-$MINCONDA_OS.sh" -O miniconda.sh; - fi - - bash miniconda.sh -b -p $HOME/miniconda - - export PATH="$HOME/miniconda/bin:$PATH" - - hash -r - - conda config --set always_yes yes --set changeps1 no - - conda update -q conda - # Useful for debugging any issues with conda - - conda info -a - - conda install pytest numpy pybind11==2.2.1 -c conda-forge - - conda install cmake gtest -c conda-forge - - conda install xtensor==0.15.1 -c QuantStack - - cmake -D BUILD_TESTS=ON -D CMAKE_INSTALL_PREFIX=$HOME/miniconda . - - make -j2 test_xtensor_python - - make install - -script: - - py.test -s - - cd test - - ./test_xtensor_python - diff --git a/CMakeLists.txt b/CMakeLists.txt index 9b5acb2..1dba9ac 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,19 +1,20 @@ ############################################################################ -# Copyright (c) 2016, Johan Mabille and Sylvain Corlay # +# Copyright (c) Wolf Vollprecht, Johan Mabille and Sylvain Corlay # +# Copyright (c) QuantStack # # # # Distributed under the terms of the BSD 3-Clause License. # # # # The full license is in the file LICENSE, distributed with this software. # ############################################################################ -cmake_minimum_required(VERSION 3.1) +cmake_minimum_required(VERSION 3.29) project(xtensor-python) set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake ${CMAKE_MODULE_PATH}) set(XTENSOR_PYTHON_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/include) -# Versionning -# =========== +# Versioning +# ========== set(XTENSOR_PYTHON_CONFIG_FILE "${XTENSOR_PYTHON_INCLUDE_DIR}/xtensor-python/xtensor_python_config.hpp") @@ -24,43 +25,70 @@ foreach(ver ${xtensor_python_version_defines}) set(XTENSOR_PYTHON_VERSION_${CMAKE_MATCH_1} "${CMAKE_MATCH_2}" CACHE INTERNAL "") endif() endforeach() -set(${PROJECT_NAME}_VERSION +set(${PROJECT_NAME}_VERSION ${XTENSOR_PYTHON_VERSION_MAJOR}.${XTENSOR_PYTHON_VERSION_MINOR}.${XTENSOR_PYTHON_VERSION_PATCH}) message(STATUS "xtensor-python v${${PROJECT_NAME}_VERSION}") # Dependencies # ============ -find_package(xtl REQUIRED) -message(STATUS "Found xtl: ${xtl_INCLUDE_DIRS}/xtl") -find_package(xtensor REQUIRED) -message(STATUS "Found xtensor: ${xtensor_INCLUDE_DIRS}/xtensor") -find_package(pybind11 REQUIRED) -message(STATUS "Found pybind11: ${pybind11_INCLUDE_DIRS}/pybind11") -find_package(NumPy REQUIRED) +set(xtensor_REQUIRED_VERSION 0.27.0) +if(TARGET xtensor) + set(xtensor_VERSION ${XTENSOR_VERSION_MAJOR}.${XTENSOR_VERSION_MINOR}.${XTENSOR_VERSION_PATCH}) + # Note: This is not SEMVER compatible comparison + if( NOT ${xtensor_VERSION} VERSION_GREATER_EQUAL ${xtensor_REQUIRED_VERSION}) + message(ERROR "Mismatch xtensor versions. Found '${xtensor_VERSION}' but requires: '${xtensor_REQUIRED_VERSION}'") + else() + message(STATUS "Found xtensor v${xtensor_VERSION}") + endif() +else() + find_package(xtensor ${xtensor_REQUIRED_VERSION} REQUIRED) + message(STATUS "Found xtensor: ${xtensor_INCLUDE_DIRS}/xtensor") +endif() + +find_package(Python COMPONENTS Interpreter REQUIRED) + +set(pybind11_REQUIRED_VERSION 3.0.0) +if (NOT TARGET pybind11::headers) + # Defaults to ON for cmake >= 3.18 + # https://github.com/pybind/pybind11/blob/35ff42b56e9d34d9a944266eb25f2c899dbdfed7/CMakeLists.txt#L96 + set(PYBIND11_FINDPYTHON OFF) + find_package(pybind11 ${pybind11_REQUIRED_VERSION} REQUIRED) + message(STATUS "Found pybind11: ${pybind11_INCLUDE_DIRS}/pybind11") +else () + # pybind11 has a variable that indicates its version already, so use that + message(STATUS "Found pybind11 v${pybind11_VERSION}") +endif () + +# Look for NumPy headers, except if NUMPY_INCLUDE_DIRS is passed, +# which is required under some circumstances (such as wasm, where +# there is no real python executable) +if(NOT NUMPY_INCLUDE_DIRS) + find_package(NumPy REQUIRED) +endif() message(STATUS "Found numpy: ${NUMPY_INCLUDE_DIRS}") # Build # ===== set(XTENSOR_PYTHON_HEADERS - ${XTENSOR_PYTHON_INCLUDE_DIR}/xtensor-python/pyarray.hpp - ${XTENSOR_PYTHON_INCLUDE_DIR}/xtensor-python/pycontainer.hpp - ${XTENSOR_PYTHON_INCLUDE_DIR}/xtensor-python/pystrides_adaptor.hpp - ${XTENSOR_PYTHON_INCLUDE_DIR}/xtensor-python/pytensor.hpp - ${XTENSOR_PYTHON_INCLUDE_DIR}/xtensor-python/pyvectorize.hpp - ${XTENSOR_PYTHON_INCLUDE_DIR}/xtensor-python/xtensor_python_config.hpp - ${XTENSOR_PYTHON_INCLUDE_DIR}/xtensor-python/xtensor_type_caster_base.hpp -) + ${XTENSOR_PYTHON_INCLUDE_DIR}/xtensor-python/pyarray.hpp + ${XTENSOR_PYTHON_INCLUDE_DIR}/xtensor-python/pyarray_backstrides.hpp + ${XTENSOR_PYTHON_INCLUDE_DIR}/xtensor-python/pycontainer.hpp + ${XTENSOR_PYTHON_INCLUDE_DIR}/xtensor-python/pynative_casters.hpp + ${XTENSOR_PYTHON_INCLUDE_DIR}/xtensor-python/pystrides_adaptor.hpp + ${XTENSOR_PYTHON_INCLUDE_DIR}/xtensor-python/pytensor.hpp + ${XTENSOR_PYTHON_INCLUDE_DIR}/xtensor-python/pyvectorize.hpp + ${XTENSOR_PYTHON_INCLUDE_DIR}/xtensor-python/xtensor_python_config.hpp + ${XTENSOR_PYTHON_INCLUDE_DIR}/xtensor-python/xtensor_type_caster_base.hpp + ) add_library(xtensor-python INTERFACE) -target_include_directories(xtensor-python INTERFACE $ - $ - $ - $) +target_include_directories(xtensor-python INTERFACE + "$" + $) target_link_libraries(xtensor-python INTERFACE xtensor) get_target_property(inc_dir xtensor-python INTERFACE_INCLUDE_DIRECTORIES) -message(STATUS "${inc_dir}") OPTION(BUILD_TESTS "xtensor test suite" OFF) OPTION(DOWNLOAD_GTEST "build gtest from downloaded sources" OFF) @@ -96,7 +124,14 @@ export(EXPORT ${PROJECT_NAME}-targets install(FILES ${XTENSOR_PYTHON_HEADERS} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/xtensor-python) -set(XTENSOR_PYTHON_CMAKECONFIG_INSTALL_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}" CACHE + +configure_file(${PROJECT_NAME}.pc.in + "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}.pc" + @ONLY) +install(FILES "${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}.pc" + DESTINATION "${CMAKE_INSTALL_DATADIR}/pkgconfig/") + +set(XTENSOR_PYTHON_CMAKECONFIG_INSTALL_DIR "${CMAKE_INSTALL_DATADIR}/cmake/${PROJECT_NAME}" CACHE STRING "install path for xtensor-pythonConfig.cmake") configure_package_config_file(${PROJECT_NAME}Config.cmake.in diff --git a/LICENSE b/LICENSE index 0e95334..a6fccbb 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,5 @@ -Copyright (c) 2016, Johan Mabille and Sylvain Corlay +Copyright (c) 2016, Wolf Vollprecht, Johan Mabille and Sylvain Corlay +Copyright (c) 2016, QuantStack All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/README.md b/README.md index 43fd54a..fbe1946 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,12 @@ -# ![xtensor-python](http://quantstack.net/assets/images/xtensor-python.svg) +# ![xtensor-python](docs/source/xtensor-python.svg) -[![Travis](https://travis-ci.org/QuantStack/xtensor-python.svg?branch=master)](https://travis-ci.org/QuantStack/xtensor-python) -[![Appveyor](https://ci.appveyor.com/api/projects/status/qx61nsg4ebxnj8s9?svg=true)](https://ci.appveyor.com/project/QuantStack/xtensor-python) +[![GHA Linux](https://github.com/xtensor-stack/xtensor/actions/workflows/linux.yml/badge.svg)](https://github.com/xtensor-stack/xtensor/actions/workflows/linux.yml) +[![GHA OSX](https://github.com/xtensor-stack/xtensor/actions/workflows/osx.yml/badge.svg)](https://github.com/xtensor-stack/xtensor/actions/workflows/osx.yml) +[![GHA Windows](https://github.com/xtensor-stack/xtensor/actions/workflows/windows.yml/badge.svg)](https://github.com/xtensor-stack/xtensor/actions/workflows/windows.yml) [![Documentation](http://readthedocs.org/projects/xtensor-python/badge/?version=latest)](https://xtensor-python.readthedocs.io/en/latest/?badge=latest) -[![Join the Gitter Chat](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/QuantStack/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) +[![Zulip](https://img.shields.io/badge/social_chat-zulip-blue.svg)](https://xtensor.zulipchat.com/#narrow/channel/539553-Ask-anything) -Python bindings for the [xtensor](https://github.com/QuantStack/xtensor) C++ multi-dimensional array library. +Python bindings for the [xtensor](https://github.com/xtensor-stack/xtensor) C++ multi-dimensional array library. - `xtensor` is a C++ library for multi-dimensional arrays enabling numpy-style broadcasting and lazy computing. - `xtensor-python` enables inplace use of numpy arrays in C++ with all the benefits from `xtensor` @@ -14,16 +15,22 @@ Python bindings for the [xtensor](https://github.com/QuantStack/xtensor) C++ mul - STL - compliant APIs. - A broad coverage of numpy APIs (see [the numpy to xtensor cheat sheet](http://xtensor.readthedocs.io/en/latest/numpy.html)). -The Python bindings for `xtensor` are based on the [pybind11](https://github.com/pybind/pybind11/) C++ library, which enables seemless interoperability between C++ and Python. +The Python bindings for `xtensor` are based on the [pybind11](https://github.com/pybind/pybind11/) C++ library, which enables seamless interoperability between C++ and Python. ## Installation -`xtensor-python` is a header-only library. We provide a package for the conda package manager. +`xtensor-python` is a header-only library. We provide a package for the mamba (or conda) package manager. ```bash -conda install -c conda-forge xtensor-python +mamba install -c conda-forge xtensor-python ``` +## Documentation + +To get started with using `xtensor-python`, check out the full documentation + +http://xtensor-python.readthedocs.io/ + ## Usage xtensor-python offers two container types wrapping numpy arrays inplace to provide an xtensor semantics @@ -43,10 +50,10 @@ Both containers enable the numpy-style APIs of xtensor (see [the numpy to xtenso ```cpp #include // Standard library import for std::accumulate -#include "pybind11/pybind11.h" // Pybind11 import to define Python bindings -#include "xtensor/xmath.hpp" // xtensor import for the C++ universal functions +#include // Pybind11 import to define Python bindings +#include // xtensor import for the C++ universal functions #define FORCE_IMPORT_ARRAY -#include "xtensor-python/pyarray.hpp" // Numpy bindings +#include // Numpy bindings double sum_of_sines(xt::pyarray& m) { @@ -71,7 +78,7 @@ import xtensor_python_test as xt v = np.arange(15).reshape(3, 5) s = xt.sum_of_sines(v) -s +print(s) ``` **Outputs** @@ -80,14 +87,22 @@ s 1.2853996391883833 ``` +**Working example** + +Get the working example here: + +* [`CMakeLists.txt`](docs/source/examples/readme_example_1/CMakeLists.txt) +* [`main.cpp`](docs/source/examples/readme_example_1/main.cpp) +* [`example.py`](docs/source/examples/readme_example_1/example.py) + ### Example 2: Create a universal function from a C++ scalar function **C++ code** ```cpp -#include "pybind11/pybind11.h" +#include #define FORCE_IMPORT_ARRAY -#include "xtensor-python/pyvectorize.hpp" +#include #include #include @@ -116,7 +131,7 @@ import xtensor_python_test as xt x = np.arange(15).reshape(3, 5) y = [1, 2, 3, 4, 5] z = xt.vectorized_func(x, y) -z +print(z) ``` **Outputs** @@ -139,7 +154,7 @@ This will pull the dependencies to xtensor-python, that is `pybind11` and `xtens ## Project cookiecutter -A template for a project making use of `xtensor-python` is available in the form of a cookiecutter [here](https://github.com/QuantStack/xtensor-python-cookiecutter). +A template for a project making use of `xtensor-python` is available in the form of a cookiecutter [here](https://github.com/xtensor-stack/xtensor-python-cookiecutter). This project is meant to help library authors get started with the xtensor python bindings. @@ -189,11 +204,18 @@ from the `docs` subdirectory. | `xtensor-python` | `xtensor` | `pybind11` | |------------------|-----------|------------------| -| master | ^0.15.1 | ~2.1.0 or ~2.2.1 | -| 0.17.x | ^0.15.1 | ~2.1.0 or ~2.2.1 | -| 0.16.x | ^0.14.0 | ~2.1.0 or ~2.2.1 | -| 0.15.x | ^0.13.1 | ~2.1.0 or ~2.2.1 | -| 0.14.x | ^0.12.0 | ~2.1.0 or ~2.2.1 | +| master | ^0.27.0 | >=2.6.1,<4 | +| 0.29.0 | ^0.27.0 | >=2.6.1,<4 | +| 0.28.0 | ^0.26.0 | >=2.6.1,<3 | +| 0.27.0 | ^0.25.0 | >=2.6.1,<3 | +| 0.26.1 | ^0.24.0 | ~2.4.3 | +| 0.26.0 | ^0.24.0 | ~2.4.3 | +| 0.25.3 | ^0.23.0 | ~2.4.3 | +| 0.25.2 | ^0.23.0 | ~2.4.3 | +| 0.25.1 | ^0.23.0 | ~2.4.3 | +| 0.25.0 | ^0.23.0 | ~2.4.3 | +| 0.24.1 | ^0.21.2 | ~2.4.3 | +| 0.24.0 | ^0.21.1 | ~2.4.3 | These dependencies are automatically resolved when using the conda package manager. diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index dfb5311..28589df 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -6,6 +6,13 @@ # The full license is in the file LICENSE, distributed with this software. # ############################################################################ +if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) + project(xtensor-python-benchmark) + + find_package(xtensor-python REQUIRED CONFIG) + set(XTENSOR_PYTHON_INCLUDE_DIR ${xtensor-python_INCLUDE_DIRS}) +endif () + message(STATUS "Forcing tests build type to Release") set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE) @@ -72,7 +79,7 @@ if (APPLE) elseif (MSVC) target_link_libraries(${XTENSOR_PYTHON_BENCHMARK_TARGET} ${PYTHON_LIBRARIES}) else () - target_link_libraries(${XTENSOR_PYTHON_BENCHMARK_TARGET} "-shared") + target_link_libraries(${XTENSOR_PYTHON_BENCHMARK_TARGET} PRIVATE xtensor-python) endif() configure_file(benchmark_pyarray.py benchmark_pyarray.py COPYONLY) @@ -81,5 +88,11 @@ configure_file(benchmark_pybind_array.py benchmark_pybind_array.py COPYONLY) configure_file(benchmark_pyvectorize.py benchmark_pyvectorize.py COPYONLY) configure_file(benchmark_pybind_vectorize.py benchmark_pybind_vectorize.py COPYONLY) -add_custom_target(xbenchmark DEPENDS ${XTENSOR_PYTHON_BENCHMARK_TARGET}) +add_custom_target(xbenchmark + COMMAND "${PYTHON_EXECUTABLE}" "benchmark_pyarray.py" + COMMAND "${PYTHON_EXECUTABLE}" "benchmark_pytensor.py" + COMMAND "${PYTHON_EXECUTABLE}" "benchmark_pybind_array.py" + COMMAND "${PYTHON_EXECUTABLE}" "benchmark_pyvectorize.py" + COMMAND "${PYTHON_EXECUTABLE}" "benchmark_pybind_vectorize.py" + DEPENDS ${XTENSOR_PYTHON_BENCHMARK_TARGET}) diff --git a/benchmark/main.cpp b/benchmark/main.cpp index c5c143b..4c18bc7 100644 --- a/benchmark/main.cpp +++ b/benchmark/main.cpp @@ -2,8 +2,8 @@ #include "pybind11/numpy.h" #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION #include "numpy/arrayobject.h" -#include "xtensor/xtensor.hpp" -#include "xtensor/xarray.hpp" +#include "xtensor/containers/xtensor.hpp" +#include "xtensor/containers/xarray.hpp" #include "xtensor-python/pyarray.hpp" #include "xtensor-python/pytensor.hpp" #include "xtensor-python/pyvectorize.hpp" @@ -17,7 +17,6 @@ PYBIND11_MODULE(benchmark_xtensor_python, m) if(_import_array() < 0) { PyErr_SetString(PyExc_ImportError, "numpy.core.multiarray failed to import"); - return nullptr; } m.doc() = "Benchmark module for xtensor python bindings"; diff --git a/benchmark/setup.py b/benchmark/setup.py index 06fe061..20633ed 100644 --- a/benchmark/setup.py +++ b/benchmark/setup.py @@ -107,7 +107,7 @@ def build_extensions(self): description='An example project using xtensor-python', long_description='', ext_modules=ext_modules, - install_requires=['pybind11==2.0.1'], + install_requires=['pybind11>=2.2.1'], cmdclass={'build_ext': BuildExt}, zip_safe=False, ) diff --git a/cmake/FindNumPy.cmake b/cmake/FindNumPy.cmake index f043566..24f3c32 100644 --- a/cmake/FindNumPy.cmake +++ b/cmake/FindNumPy.cmake @@ -40,9 +40,9 @@ # Finding NumPy involves calling the Python interpreter if(NumPy_FIND_REQUIRED) - find_package(PythonInterp REQUIRED) + find_package(Python COMPONENTS Interpreter REQUIRED) else() - find_package(PythonInterp) + find_package(Python COMPONENTS Interpreter) endif() if(NOT PYTHONINTERP_FOUND) diff --git a/docs/environment.yml b/docs/environment.yml index 47e93f7..1e6e74a 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -5,3 +5,4 @@ channels: dependencies: - breathe + - sphinx_rtd_theme diff --git a/docs/source/_static/main_stylesheet.css b/docs/source/_static/main_stylesheet.css new file mode 100644 index 0000000..dc4199e --- /dev/null +++ b/docs/source/_static/main_stylesheet.css @@ -0,0 +1,4 @@ +.wy-nav-content{ + max-width: 1000px; + margin: auto; +} diff --git a/docs/source/basic_usage.rst b/docs/source/basic_usage.rst index c97c9ac..32b1c23 100644 --- a/docs/source/basic_usage.rst +++ b/docs/source/basic_usage.rst @@ -16,7 +16,7 @@ Example 1: Use an algorithm of the C++ library on a numpy array inplace #include // Standard library import for std::accumulate #include "pybind11/pybind11.h" // Pybind11 import to define Python bindings - #include "xtensor/xmath.hpp" // xtensor import for the C++ universal functions + #include "xtensor/core/xmath.hpp" // xtensor import for the C++ universal functions #define FORCE_IMPORT_ARRAY // numpy C api loading #include "xtensor-python/pyarray.hpp" // Numpy bindings diff --git a/docs/source/compilers.rst b/docs/source/compilers.rst new file mode 100644 index 0000000..62ef028 --- /dev/null +++ b/docs/source/compilers.rst @@ -0,0 +1,20 @@ +.. Copyright (c) 2016, Johan Mabille, Sylvain Corlay and Wolf Vollprecht + + Distributed under the terms of the BSD 3-Clause License. + + The full license is in the file LICENSE, distributed with this software. + +Compiler workarounds +==================== + +This page tracks the workarounds for the various compiler issues that we +encountered in the development. This is mostly of interest for developers +interested in contributing to xtensor-python. + +GCC and ``std::allocator`` +------------------------------------- + +GCC sometimes fails to automatically instantiate the ``std::allocator`` +class template for the types ``long long`` and ``unsigned long long``. +Those allocators are thus explicitly instantiated in the dummy function +``void long_long_allocator()`` in the file ``py_container.hpp``. diff --git a/docs/source/conf.py b/docs/source/conf.py index 5032a79..7fcaddc 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -15,9 +15,13 @@ html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] -extensions = ['breathe'] +def setup(app): + app.add_css_file("main_stylesheet.css") + +extensions = ['breathe', 'sphinx_rtd_theme'] breathe_projects = { 'xtensor-python': '../xml' } templates_path = ['_templates'] +html_static_path = ['_static'] source_suffix = '.rst' master_doc = 'index' project = 'xtensor-python' diff --git a/docs/source/cookiecutter.rst b/docs/source/cookiecutter.rst index e326d13..e725106 100644 --- a/docs/source/cookiecutter.rst +++ b/docs/source/cookiecutter.rst @@ -28,11 +28,11 @@ Install cookiecutter_ pip install cookiecutter -After installing cookiecutter, use the xtensor_cookiecutter_: +After installing cookiecutter, use the `xtensor-python-cookiecutter`_: .. code:: - cookiecutter https://github.com/QuantStack/xtensor-python-cookiecutter.git + cookiecutter https://github.com/xtensor-stack/xtensor-python-cookiecutter.git As xtensor-python-cookiecutter runs, you will be asked for basic information about your custom extension project. You will be prompted for the following @@ -50,5 +50,6 @@ This will produce a directory containing all the required content for a minimal project making use of xtensor with all the required boilerplate for package management, together with a few basic examples. -.. _xtensor_cookicutter: https://github.com/QuantStack/xtensor-python-cookiecutter +.. _xtensor-python-cookiecutter: https://github.com/xtensor-stack/xtensor-python-cookiecutter + .. _cookiecutter: https://github.com/audreyr/cookiecutter diff --git a/docs/source/dev_build_options.rst b/docs/source/dev_build_options.rst new file mode 100644 index 0000000..80e7e78 --- /dev/null +++ b/docs/source/dev_build_options.rst @@ -0,0 +1,45 @@ +.. Copyright (c) 2016, Johan Mabille and Sylvain Corlay + + Distributed under the terms of the BSD 3-Clause License. + + The full license is in the file LICENSE, distributed with this software. + + +Build, test and benchmark +========================= + +``xtensor-python`` build supports the following options: + +- ``BUILD_TESTS``: enables the ``xtest`` and ``xbenchmark`` targets (see below). +- ``DOWNLOAD_GTEST``: downloads ``gtest`` and builds it locally instead of using a binary installation. +- ``GTEST_SRC_DIR``: indicates where to find the ``gtest`` sources instead of downloading them. + +All these options are disabled by default. Enabling ``DOWNLOAD_GTEST`` or +setting ``GTEST_SRC_DIR`` enables ``BUILD_TESTS``. + +If the ``BUILD_TESTS`` option is enabled, the following targets are available: + +- xtest: builds an run the test suite. +- xbenchmark: builds and runs the benchmarks. + +For instance, building the test suite of ``xtensor-python`` and downloading ``gtest`` automatically: + +.. code:: + + mkdir build + cd build + cmake -DDOWNLOAD_GTEST=ON ../ + make xtest + +To run the benchmark: + +.. code:: + + make xbenchmark + +To test the Python bindings: + +.. code:: + + cd .. + pytest -s diff --git a/docs/source/examples.rst b/docs/source/examples.rst new file mode 100644 index 0000000..6c9def8 --- /dev/null +++ b/docs/source/examples.rst @@ -0,0 +1,196 @@ + +**************** +(CMake) Examples +**************** + +Basic example (from readme) +=========================== + +Consider the following C++ code: + +:download:`main.cpp ` + +.. literalinclude:: examples/readme_example_1/main.cpp + :language: cpp + +There are several options to build the module, +whereby we will use *CMake* here with the following ``CMakeLists.txt``: + +:download:`CMakeLists.txt ` + +.. literalinclude:: examples/readme_example_1/CMakeLists.txt + :language: cmake + +.. tip:: + + There is a potential pitfall here, centered around the fact that *CMake* + has a 'new' *FindPython* and a 'classic' *FindPythonLibs*. + We here use *FindPython* because of its ability to find the NumPy headers, + that we need for *xtensor-python*. + + This has the consequence that when we want to force *CMake* + to use a specific *Python* executable, we have to use something like + + .. code-block:: none + + cmake -Bbuild -DPython_EXECUTABLE=`which python` + + whereby it is crucial that one uses the correct case ``Python_EXECUTABLE``, as: + + .. code-block:: none + + Python_EXECUTABLE <-> FindPython + PYTHON_EXECUTABLE <-> FindPythonLibs + + (remember that *CMake* is **case-sensitive**!). + + Now, since we use *FindPython* because of *xtensor-python* we also want *pybind11* + to use *FindPython* + (and not the classic *FindPythonLibs*, + since we want to specify the *Python* executable only once). + To this end we have to make sure to do things in the correct order, which is + + .. code-block:: cmake + + find_package(Python REQUIRED COMPONENTS Interpreter Development NumPy) + find_package(pybind11 REQUIRED CONFIG) + + (i.e. one finds *Python* **before** *pybind11*). + See the `pybind11 documentation `_. + + In addition, be sure to use a quite recent *CMake* version, + by starting your ``CMakeLists.txt`` for example with + + .. code-block:: cmake + + cmake_minimum_required(VERSION 3.18..3.20) + +Then we can test the module: + +:download:`example.py ` + +.. literalinclude:: examples/readme_example_1/example.py + :language: cmake + +.. note:: + + Since we did not install the module, + we should compile and run the example from the same folder. + To install, please consult + `this *pybind11* / *CMake* example `_. + + +Type restriction with SFINAE +============================ + +.. seealso:: + + `Medium post by Johan Mabille `__ + This example covers "Option 4". + +In this example we will design a module with a function that accepts an ``xt::xtensor`` as argument, +but in such a way that an ``xt::pyxtensor`` can be accepted in the Python module. +This is done by having a templated function + +.. code-block:: cpp + + template + void times_dimension(T& t); + +As this might be a bit too permissive for your liking, we will show you how to limit the +scope to *xtensor* types, and allow other overloads using the principle of SFINAE +(Substitution Failure Is Not An Error). +In particular: + +:download:`mymodule.hpp ` + +.. literalinclude:: examples/sfinae/mymodule.hpp + :language: cpp + +Consequently from C++, the interaction with the module's function is trivial + +:download:`main.cpp ` + +.. literalinclude:: examples/sfinae/main.cpp + :language: cpp + +For the Python module we just have to specify the template to be +``xt::pyarray`` or ``xt::pytensor``. E.g. + +:download:`src/python.cpp ` + +.. literalinclude:: examples/sfinae/python.cpp + :language: cpp + +We will again use *CMake* to compile, with the following ``CMakeLists.txt``: + +:download:`CMakeLists.txt ` + +.. literalinclude:: examples/sfinae/CMakeLists.txt + :language: cmake + +(see *CMake* tip above). + +Then we can test the module: + +:download:`example.py ` + +.. literalinclude:: examples/readme_example_1/example.py + :language: cmake + +.. note:: + + Since we did not install the module, + we should compile and run the example from the same folder. + To install, please consult + `this pybind11 / CMake example `_. + **Tip**: take care to modify that example with the correct *CMake* case ``Python_EXECUTABLE``. + +Fall-back cast +============== + +The previous example showed you how to design your module to be flexible in accepting data. +From C++ we used ``xt::xarray``, +whereas for the Python API we used ``xt::pyarray`` to operate directly on the memory +of a NumPy array from Python (without copying the data). + +Sometimes, you might not have the flexibility to design your module's methods +with template parameters. +This might occur when you want to ``override`` functions +(though it is recommended to use CRTP to still use templates). +In this case we can still bind the module in Python using *xtensor-python*, +however, we have to copy the data from a (NumPy) array. +This means that although the following signatures are quite different when used from C++, +as follows: + +1. *Constant reference*: read from the data, without copying it. + + .. code-block:: cpp + + void foo(const xt::xarray& a); + +2. *Reference*: read from and/or write to the data, without copying it. + + .. code-block:: cpp + + void foo(xt::xarray& a); + +3. *Copy*: copy the data. + + .. code-block:: cpp + + void foo(xt::xarray a); + +The Python will all cases result in a copy to a temporary variable +(though the last signature will lead to a copy to a temporary variable, and another copy to ``a``). +On the one hand, this is more costly than when using ``xt::pyarray`` and ``xt::pyxtensor``, +on the other hand, it means that all changes you make to a reference, are made to the temporary +copy, and are thus lost. + +Still, it might be a convenient way to create Python bindings, using a minimal effort. +Consider this example: + +:download:`main.cpp ` + +.. literalinclude:: examples/copy_cast/main.cpp + :language: cpp diff --git a/docs/source/examples/copy_cast/CMakeLists.txt b/docs/source/examples/copy_cast/CMakeLists.txt new file mode 100644 index 0000000..d8daf2a --- /dev/null +++ b/docs/source/examples/copy_cast/CMakeLists.txt @@ -0,0 +1,13 @@ +cmake_minimum_required(VERSION 3.29) + +project(mymodule) + +find_package(pybind11 CONFIG REQUIRED) +find_package(xtensor REQUIRED) +find_package(xtensor-python REQUIRED) +find_package(Python REQUIRED COMPONENTS NumPy) + +pybind11_add_module(mymodule main.cpp) +target_link_libraries(mymodule PUBLIC pybind11::module xtensor-python Python::NumPy) + +target_compile_definitions(mymodule PRIVATE VERSION_INFO=0.1.0) diff --git a/docs/source/examples/copy_cast/example.py b/docs/source/examples/copy_cast/example.py new file mode 100644 index 0000000..d007e7c --- /dev/null +++ b/docs/source/examples/copy_cast/example.py @@ -0,0 +1,6 @@ +import mymodule +import numpy as np + +c = np.array([[1, 2, 3], [4, 5, 6]]) +assert np.isclose(np.sum(np.sin(c)), mymodule.sum_of_sines(c)) +assert np.isclose(np.sum(np.cos(c)), mymodule.sum_of_cosines(c)) diff --git a/docs/source/examples/copy_cast/main.cpp b/docs/source/examples/copy_cast/main.cpp new file mode 100644 index 0000000..2e12609 --- /dev/null +++ b/docs/source/examples/copy_cast/main.cpp @@ -0,0 +1,27 @@ +#include +#include +#include +#define FORCE_IMPORT_ARRAY +#include + +template +double sum_of_sines(T& m) +{ + auto sines = xt::sin(m); // sines does not actually hold values. + return std::accumulate(sines.begin(), sines.end(), 0.0); +} + +// In the Python API this a reference to a temporary variable +double sum_of_cosines(const xt::xarray& m) +{ + auto cosines = xt::cos(m); // cosines does not actually hold values. + return std::accumulate(cosines.begin(), cosines.end(), 0.0); +} + +PYBIND11_MODULE(mymodule, m) +{ + xt::import_numpy(); + m.doc() = "Test module for xtensor python bindings"; + m.def("sum_of_sines", sum_of_sines>, "Sum the sines of the input values"); + m.def("sum_of_cosines", sum_of_cosines, "Sum the cosines of the input values"); +} diff --git a/docs/source/examples/readme_example_1/CMakeLists.txt b/docs/source/examples/readme_example_1/CMakeLists.txt new file mode 100644 index 0000000..8831bca --- /dev/null +++ b/docs/source/examples/readme_example_1/CMakeLists.txt @@ -0,0 +1,13 @@ +cmake_minimum_required(VERSION 3.29) + +project(mymodule) + +find_package(Python REQUIRED COMPONENTS Interpreter Development NumPy) +find_package(pybind11 REQUIRED CONFIG) +find_package(xtensor REQUIRED) +find_package(xtensor-python REQUIRED) + +pybind11_add_module(mymodule main.cpp) +target_link_libraries(mymodule PUBLIC pybind11::module xtensor-python Python::NumPy) + +target_compile_definitions(mymodule PRIVATE VERSION_INFO=0.1.0) diff --git a/docs/source/examples/readme_example_1/example.py b/docs/source/examples/readme_example_1/example.py new file mode 100644 index 0000000..1ae033d --- /dev/null +++ b/docs/source/examples/readme_example_1/example.py @@ -0,0 +1,6 @@ +import mymodule +import numpy as np + +a = np.array([1, 2, 3]) +assert np.isclose(np.sum(np.sin(a)), mymodule.sum_of_sines(a)) + diff --git a/docs/source/examples/readme_example_1/main.cpp b/docs/source/examples/readme_example_1/main.cpp new file mode 100644 index 0000000..6175ed8 --- /dev/null +++ b/docs/source/examples/readme_example_1/main.cpp @@ -0,0 +1,18 @@ +#include +#include +#include +#define FORCE_IMPORT_ARRAY +#include + +double sum_of_sines(xt::pyarray& m) +{ + auto sines = xt::sin(m); // sines does not actually hold values. + return std::accumulate(sines.begin(), sines.end(), 0.0); +} + +PYBIND11_MODULE(mymodule, m) +{ + xt::import_numpy(); + m.doc() = "Test module for xtensor python bindings"; + m.def("sum_of_sines", sum_of_sines, "Sum the sines of the input values"); +} diff --git a/docs/source/examples/sfinae/CMakeLists.txt b/docs/source/examples/sfinae/CMakeLists.txt new file mode 100644 index 0000000..1fb8477 --- /dev/null +++ b/docs/source/examples/sfinae/CMakeLists.txt @@ -0,0 +1,16 @@ +cmake_minimum_required(VERSION 3.29) + +project(mymodule) + +find_package(Python REQUIRED COMPONENTS Interpreter Development NumPy) +find_package(pybind11 REQUIRED CONFIG) +find_package(xtensor REQUIRED) +find_package(xtensor-python REQUIRED) + +pybind11_add_module(mymodule python.cpp) +target_link_libraries(mymodule PUBLIC pybind11::module xtensor-python Python::NumPy) + +target_compile_definitions(mymodule PRIVATE VERSION_INFO=0.1.0) + +add_executable(myexec main.cpp) +target_link_libraries(myexec PUBLIC xtensor) diff --git a/docs/source/examples/sfinae/example.py b/docs/source/examples/sfinae/example.py new file mode 100644 index 0000000..1929f0b --- /dev/null +++ b/docs/source/examples/sfinae/example.py @@ -0,0 +1,8 @@ +import mymodule +import numpy as np + +a = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float64) +b = np.array(a, copy=True) +mymodule.times_dimension(b) # changing in-place! +assert np.allclose(2 * a, b) + diff --git a/docs/source/examples/sfinae/main.cpp b/docs/source/examples/sfinae/main.cpp new file mode 100644 index 0000000..44ffaa7 --- /dev/null +++ b/docs/source/examples/sfinae/main.cpp @@ -0,0 +1,10 @@ +#include "mymodule.hpp" +#include + +int main() +{ + xt::xtensor a = xt::arange(2 * 3).reshape({2, 3}); + mymodule::times_dimension(a); + std::cout << a << std::endl; + return 0; +} diff --git a/docs/source/examples/sfinae/mymodule.hpp b/docs/source/examples/sfinae/mymodule.hpp new file mode 100644 index 0000000..33f7b8a --- /dev/null +++ b/docs/source/examples/sfinae/mymodule.hpp @@ -0,0 +1,32 @@ +#include + +namespace mymodule { + +template +struct is_std_vector +{ + static const bool value = false; +}; + +template +struct is_std_vector > +{ + static const bool value = true; +}; + +// any xtensor object +template ::value, bool> = true> +void times_dimension(T& t) +{ + using value_type = typename T::value_type; + t *= (value_type)(t.dimension()); +} + +// an std::vector +template ::value, bool> = true> +void times_dimension(T& t) +{ + // do nothing +} + +} diff --git a/docs/source/examples/sfinae/python.cpp b/docs/source/examples/sfinae/python.cpp new file mode 100644 index 0000000..081f625 --- /dev/null +++ b/docs/source/examples/sfinae/python.cpp @@ -0,0 +1,11 @@ +#include "mymodule.hpp" +#include +#define FORCE_IMPORT_ARRAY +#include + +PYBIND11_MODULE(mymodule, m) +{ + xt::import_numpy(); + m.doc() = "Test module for xtensor python bindings"; + m.def("times_dimension", &mymodule::times_dimension>); +} diff --git a/docs/source/index.rst b/docs/source/index.rst index 72ff792..d54686b 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -16,7 +16,7 @@ What are ``xtensor`` and ``xtensor-python``? - ``xtensor`` is a C++ library for multi-dimensional arrays enabling numpy-style broadcasting and lazy computing. - ``xtensor-python`` enables inplace use of numpy arrays with all the benefits from ``xtensor`` - - C++ universal functions and broadcasting + - C++ universal functions and broadcasting - STL - compliant APIs. @@ -62,6 +62,7 @@ This software is licensed under the BSD-3-Clause license. See the LICENSE file f basic_usage array_tensor numpy_capi + examples cookiecutter .. toctree:: @@ -73,11 +74,13 @@ This software is licensed under the BSD-3-Clause license. See the LICENSE file f .. toctree:: :caption: DEVELOPER ZONE + dev_build_options + compilers releasing .. _NumPy: http://www.numpy.org .. _`Buffer Protocol`: https://docs.python.org/3/c-api/buffer.html -.. _`numpy to xtensor cheat sheet`: http://xtensor.readthedocs.io/en/latest/numpy.html -.. _xtensor: https://github.com/QuantStack/xtensor -.. _pybind11: https://github.com/pybind/pybind11 -.. _xtensor-python-cookiecutter: https://github.com/QuantStack/xtensor-python-cookiecutter +.. _`numpy to xtensor cheat sheet`: http://xtensor.readthedocs.io/en/latest/numpy.html +.. _xtensor: https://github.com/xtensor-stack/xtensor +.. _pybind11: https://github.com/pybind/pybind11 +.. _xtensor-python-cookiecutter: https://github.com/xtensor-stack/xtensor-python-cookiecutter diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 2965712..d37d3e9 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -27,14 +27,14 @@ Besides the xtendor-python headers, all these methods place the `cmake` project .. image:: conda.svg -Using the conda package ------------------------ +Using the conda-forge package +---------------------------- -A package for xtensor-python is available on the conda package manager. +A package for xtensor-python is available on the mamba (or conda) package manager. .. code:: - conda install -c conda-forge xtensor-python + mamba install -c conda-forge xtensor-python .. image:: debian.svg diff --git a/docs/source/numpy_capi.rst b/docs/source/numpy_capi.rst index aa0855f..d489be2 100644 --- a/docs/source/numpy_capi.rst +++ b/docs/source/numpy_capi.rst @@ -23,7 +23,6 @@ Thus the basic skeleton of the module looks like: .. code:: - #include "pybind11/pybind11.h" #define FORCE_IMPORT_ARRAY #include "xtensor-python/pyarray.hpp" @@ -37,9 +36,45 @@ Thus the basic skeleton of the module looks like: Extension module with multiple files ------------------------------------ -If the extension module contain many source files that include ``xtensor-python`` header files, the previous points are still -required. However, the source files that don't contain the initializing code of the module can directly include ``xtensor-python`` -header files. +If the extension module contains many source files that include ``xtensor-python`` header files, the previous points are still +required. However, the symbol ``FORCE_IMPORT_ARRAY`` must be defined only once. The simplest is to define it int the file that +contains the initializing code of the module, you can then directly include ``xtensor-python`` headers in other files. Let's +illustrate this with an extension modules containing the following files: + +- ``main.cpp``: initializing code of the module +- ``image.hpp``: declaration of the ``image`` class embedding an ``xt::pyarray`` object +- ``image.cpp``: implementation of the ``image`` class + +The basic skeleton of the module looks like: + +.. code:: + + // image.hpp + // Do NOT define FORCE_IMPORT_ARRAY here + #include "xtensor-python/pyarray.hpp" + + class image + { + // .... + private: + xt::pyarray m_data; + }; + + // image.cpp + // Do NOT define FORCE_IMPORT_ARRAY here + #include "image.hpp" + // definition of the image class + + // main.cpp + // FORCE_IMPORT_ARRAY must be define ONCE, BEFORE including + // any header from xtensor-python (even indirectly) + #define FORCE_IMPORT_ARRAY + #include "image.hpp" + PYBIND11_MODULE(plugin_name, m) + { + xt::import_numpy(); + //... + } Using other extension modules diff --git a/environment-dev.yml b/environment-dev.yml new file mode 100644 index 0000000..4d3b27b --- /dev/null +++ b/environment-dev.yml @@ -0,0 +1,15 @@ +name: xtensor-python +channels: + - conda-forge +dependencies: + # Build dependencies + - cmake + - ninja + # Host dependencies + - xtensor>=0.27,<0.28 + - numpy>=2.0 + - pybind11>=2.12.0,<4 + # Test dependencies + - setuptools + - pytest + diff --git a/include/xtensor-python/pyarray.hpp b/include/xtensor-python/pyarray.hpp index baf7fdc..c24a793 100644 --- a/include/xtensor-python/pyarray.hpp +++ b/include/xtensor-python/pyarray.hpp @@ -1,5 +1,6 @@ /*************************************************************************** -* Copyright (c) 2016, Johan Mabille and Sylvain Corlay * +* Copyright (c) Wolf Vollprecht, Johan Mabille and Sylvain Corlay * +* Copyright (c) QuantStack * * * * Distributed under the terms of the BSD 3-Clause License. * * * @@ -13,17 +14,20 @@ #include #include -#include "xtensor/xbuffer_adaptor.hpp" -#include "xtensor/xiterator.hpp" -#include "xtensor/xsemantic.hpp" +#include "xtensor/containers/xbuffer_adaptor.hpp" +#include "xtensor/core/xiterator.hpp" +#include "xtensor/core/xsemantic.hpp" +#include "pyarray_backstrides.hpp" #include "pycontainer.hpp" #include "pystrides_adaptor.hpp" +#include "pynative_casters.hpp" #include "xtensor_type_caster_base.hpp" +#include "xtensor_python_config.hpp" namespace xt { - template + template class pyarray; } @@ -31,30 +35,27 @@ namespace pybind11 { namespace detail { - template - struct handle_type_name> +#ifdef PYBIND11_DESCR // The macro is removed from pybind11 since 2.3 + template + struct handle_type_name> { static PYBIND11_DESCR name() { - return _("numpy.ndarray[") + make_caster::name() + _("]"); + return _("numpy.ndarray[") + npy_format_descriptor::name() + _("]"); } }; +#endif - template - struct pyobject_caster> + template + struct pyobject_caster> { - using type = xt::pyarray; + using type = xt::pyarray; bool load(handle src, bool convert) { if (!convert) { - if (!PyArray_Check(src.ptr())) - { - return false; - } - int type_num = xt::detail::numpy_traits::type_num; - if(xt::detail::pyarray_type(reinterpret_cast(src.ptr())) != type_num) + if (!xt::detail::check_array(src)) { return false; } @@ -68,14 +69,18 @@ namespace pybind11 return src.inc_ref(); } +#ifdef PYBIND11_DESCR // The macro is removed from pybind11 since 2.3 PYBIND11_TYPE_CASTER(type, handle_type_name::name()); +#else + PYBIND11_TYPE_CASTER(type, _("numpy.ndarray[") + npy_format_descriptor::name + _("]")); +#endif }; // Type caster for casting ndarray to xexpression - template - struct type_caster>> : pyobject_caster> + template + struct type_caster>> : pyobject_caster> { - using Type = xt::xexpression>; + using Type = xt::xexpression>; operator Type&() { @@ -88,251 +93,69 @@ namespace pybind11 } }; - // Type caster for casting xarray to ndarray - template - struct type_caster> : xtensor_type_caster_base> - { - }; } } namespace xt { - - /************************** - * pybackstrides_iterator * - **************************/ - - template - class pybackstrides_iterator + template + struct xiterable_inner_types> + : xcontainer_iterable_types> { - public: - - using self_type = pybackstrides_iterator; - - using value_type = typename B::value_type; - using pointer = const value_type*; - using reference = value_type; - using difference_type = std::ptrdiff_t; - using iterator_category = std::random_access_iterator_tag; - - inline pybackstrides_iterator(const B* b, std::size_t offset) - : p_b(b), m_offset(offset) - { - } - - inline reference operator*() const - { - return p_b->operator[](m_offset); - } - - inline pointer operator->() const - { - // Returning the address of a temporary - value_type res = p_b->operator[](m_offset); - return &res; - } - - inline reference operator[](difference_type n) const - { - return p_b->operator[](m_offset + n); - } - - inline self_type& operator++() - { - ++m_offset; - return *this; - } - - inline self_type& operator--() - { - --m_offset; - return *this; - } - - inline self_type operator++(int) - { - self_type tmp(*this); - ++m_offset; - return tmp; - } - - inline self_type operator--(int) - { - self_type tmp(*this); - --m_offset; - return tmp; - } - - inline self_type& operator+=(difference_type n) - { - m_offset += n; - return *this; - } - - inline self_type& operator-=(difference_type n) - { - m_offset -= n; - return *this; - } - - inline self_type operator+(difference_type n) const - { - return self_type(p_b, m_offset + n); - } - - inline self_type operator-(difference_type n) const - { - return self_type(p_b, m_offset - n); - } - - inline self_type operator-(const self_type& rhs) const - { - self_type tmp(*this); - tmp -= (m_offset - rhs.m_offset); - return tmp; - } - - inline std::size_t offset() const - { - return m_offset; - } - - private: - - const B* p_b; - std::size_t m_offset; }; - template - inline bool operator==(const pybackstrides_iterator& lhs, - const pybackstrides_iterator& rhs) - { - return lhs.offset() == rhs.offset(); - } - - template - inline bool operator!=(const pybackstrides_iterator& lhs, - const pybackstrides_iterator& rhs) - { - return !(lhs == rhs); - } - - template - inline bool operator<(const pybackstrides_iterator& lhs, - const pybackstrides_iterator& rhs) - { - return lhs.offset() < rhs.offset(); - } - - template - inline bool operator<=(const pybackstrides_iterator& lhs, - const pybackstrides_iterator& rhs) - { - return (lhs < rhs) || (lhs == rhs); - } - - template - inline bool operator>(const pybackstrides_iterator& lhs, - const pybackstrides_iterator& rhs) + template + struct xcontainer_inner_types> { - return !(lhs <= rhs); - } - - template - inline bool operator>=(const pybackstrides_iterator& lhs, - const pybackstrides_iterator& rhs) - { - return !(lhs < rhs); - } - - template - class pyarray_backstrides - { - public: - - using self_type = pyarray_backstrides; - using array_type = A; - using value_type = typename array_type::size_type; - using const_reference = value_type; - using const_pointer = const value_type*; - using size_type = typename array_type::size_type; - using difference_type = typename array_type::difference_type; - - using const_iterator = pybackstrides_iterator; - - pyarray_backstrides() = default; - pyarray_backstrides(const array_type& a); - - bool empty() const; - size_type size() const; - - value_type operator[](size_type i) const; - - const_reference front() const; - const_reference back() const; - - const_iterator begin() const; - const_iterator end() const; - const_iterator cbegin() const; - const_iterator cend() const; - - private: - - const array_type* p_a; - }; - - template - struct xiterable_inner_types> - : xcontainer_iterable_types> - { - }; - - template - struct xcontainer_inner_types> - { - using container_type = xbuffer_adaptor; - using shape_type = std::vector; - using strides_type = shape_type; - using backstrides_type = pyarray_backstrides>; + using storage_type = xbuffer_adaptor; + using reference = typename storage_type::reference; + using const_reference = typename storage_type::const_reference; + using size_type = typename storage_type::size_type; + using shape_type = std::vector; + using strides_type = std::vector; + using backstrides_type = pyarray_backstrides>; using inner_shape_type = xbuffer_adaptor; using inner_strides_type = pystrides_adaptor; using inner_backstrides_type = backstrides_type; - using temporary_type = pyarray; - static constexpr layout_type layout = layout_type::dynamic; + using temporary_type = pyarray; + static constexpr layout_type layout = L; }; /** * @class pyarray * @brief Multidimensional container providing the xtensor container semantics to a numpy array. * - * pyarray is similar to the xarray container in that it has a dynamic dimensionality. Reshapes of - * a pyarray container are reflected in the underlying numpy array. + * pyarray is similar to the xarray container in that it has a dynamic dimensionality. + * Reshapes of a pyarray container are reflected in the underlying numpy array. * * @tparam T The type of the element stored in the pyarray. + * @tparam L Static layout of the pyarray + * * @sa pytensor */ - template - class pyarray : public pycontainer>, - public xcontainer_semantic> + template + class pyarray : public pycontainer>, + public xcontainer_semantic> { public: - using self_type = pyarray; + using self_type = pyarray; using semantic_base = xcontainer_semantic; using base_type = pycontainer; - using container_type = typename base_type::container_type; + using storage_type = typename base_type::storage_type; using value_type = typename base_type::value_type; using reference = typename base_type::reference; using const_reference = typename base_type::const_reference; using pointer = typename base_type::pointer; using size_type = typename base_type::size_type; + using difference_type = typename base_type::difference_type; using shape_type = typename base_type::shape_type; using strides_type = typename base_type::strides_type; using backstrides_type = typename base_type::backstrides_type; using inner_shape_type = typename base_type::inner_shape_type; using inner_strides_type = typename base_type::inner_strides_type; using inner_backstrides_type = typename base_type::inner_backstrides_type; + constexpr static std::size_t rank = SIZE_MAX; pyarray(); pyarray(const value_type& t); @@ -351,6 +174,9 @@ namespace xt explicit pyarray(const shape_type& shape, const strides_type& strides, const_reference value); explicit pyarray(const shape_type& shape, const strides_type& strides); + template + static pyarray from_shape(S&& s); + pyarray(const self_type& rhs); self_type& operator=(const self_type& rhs); @@ -369,12 +195,26 @@ namespace xt static self_type ensure(pybind11::handle h); static bool check_(pybind11::handle h); +#if (PYBIND11_VERSION_MAJOR == 2 && PYBIND11_VERSION_MINOR >= 3) || PYBIND11_VERSION_MAJOR >= 3 + // Prevent ambiguous overload resolution for operators defined for + // both xt::xcontainer_semantic and pybind11::object. + using semantic_base::operator+=; + using semantic_base::operator-=; + using semantic_base::operator*=; + using semantic_base::operator/=; + using semantic_base::operator|=; + using semantic_base::operator&=; + using semantic_base::operator^=; + // using semantic_base::operator<<=; + // using semantic_base::operator>>=; +#endif + private: inner_shape_type m_shape; inner_strides_type m_strides; mutable inner_backstrides_type m_backstrides; - container_type m_data; + storage_type m_storage; void init_array(const shape_type& shape, const strides_type& strides); void init_from_python(); @@ -383,83 +223,14 @@ namespace xt const inner_strides_type& strides_impl() const noexcept; const inner_backstrides_type& backstrides_impl() const noexcept; - container_type& data_impl() noexcept; - const container_type& data_impl() const noexcept; - - friend class xcontainer>; - friend class pycontainer>; - }; - - /************************************** - * pyarray_backstrides implementation * - **************************************/ - - template - inline pyarray_backstrides::pyarray_backstrides(const array_type& a) - : p_a(&a) - { - } - - template - inline bool pyarray_backstrides::empty() const - { - return p_a->dimension() == 0; - } - - template - inline auto pyarray_backstrides::size() const -> size_type - { - return p_a->dimension(); - } - - template - inline auto pyarray_backstrides::operator[](size_type i) const -> value_type - { - value_type sh = p_a->shape()[i]; - value_type res = sh == 1 ? 0 : (sh - 1) * p_a->strides()[i]; - return res; - } - - template - inline auto pyarray_backstrides::front() const -> const_reference - { - value_type sh = p_a->shape()[0]; - value_type res = sh == 1 ? 0 : (sh - 1) * p_a->strides()[0]; - return res; - } - - template - inline auto pyarray_backstrides::back() const -> const_reference - { - auto index = p_a->size() - 1; - value_type sh = p_a->shape()[index]; - value_type res = sh == 1 ? 0 : (sh - 1) * p_a->strides()[index]; - return res; - } + storage_type& storage_impl() noexcept; + const storage_type& storage_impl() const noexcept; - template - inline auto pyarray_backstrides::begin() const -> const_iterator - { - return cbegin(); - } + layout_type default_dynamic_layout(); - template - inline auto pyarray_backstrides::end() const -> const_iterator - { - return cend(); - } - - template - inline auto pyarray_backstrides::cbegin() const -> const_iterator - { - return const_iterator(this, 0); - } - - template - inline auto pyarray_backstrides::cend() const -> const_iterator - { - return const_iterator(this, size()); - } + friend class xcontainer>; + friend class pycontainer>; + }; /************************** * pyarray implementation * @@ -469,84 +240,84 @@ namespace xt * @name Constructors */ //@{ - template - inline pyarray::pyarray() + template + inline pyarray::pyarray() : base_type() { // TODO: avoid allocation shape_type shape = xtl::make_sequence(0, size_type(1)); strides_type strides = xtl::make_sequence(0, size_type(0)); init_array(shape, strides); - m_data[0] = T(); + detail::default_initialize(m_storage); } /** * Allocates a pyarray with nested initializer lists. */ - template - inline pyarray::pyarray(const value_type& t) + template + inline pyarray::pyarray(const value_type& t) : base_type() { - base_type::resize(xt::shape(t), layout_type::row_major); - nested_copy(m_data.begin(), t); + base_type::resize(xt::shape(t), default_dynamic_layout()); + nested_copy(m_storage.begin(), t); } - template - inline pyarray::pyarray(nested_initializer_list_t t) + template + inline pyarray::pyarray(nested_initializer_list_t t) : base_type() { - base_type::resize(xt::shape(t), layout_type::row_major); - nested_copy(m_data.begin(), t); + base_type::resize(xt::shape(t), default_dynamic_layout()); + L == layout_type::row_major ? nested_copy(m_storage.begin(), t) : nested_copy(this->template begin(), t); } - template - inline pyarray::pyarray(nested_initializer_list_t t) + template + inline pyarray::pyarray(nested_initializer_list_t t) : base_type() { - base_type::resize(xt::shape(t), layout_type::row_major); - nested_copy(m_data.begin(), t); + base_type::resize(xt::shape(t), default_dynamic_layout()); + L == layout_type::row_major ? nested_copy(m_storage.begin(), t) : nested_copy(this->template begin(), t); } - template - inline pyarray::pyarray(nested_initializer_list_t t) + template + inline pyarray::pyarray(nested_initializer_list_t t) : base_type() { - base_type::resize(xt::shape(t), layout_type::row_major); - nested_copy(m_data.begin(), t); + base_type::resize(xt::shape(t), default_dynamic_layout()); + L == layout_type::row_major ? nested_copy(m_storage.begin(), t) : nested_copy(this->template begin(), t); } - template - inline pyarray::pyarray(nested_initializer_list_t t) + template + inline pyarray::pyarray(nested_initializer_list_t t) : base_type() { - base_type::resize(xt::shape(t), layout_type::row_major); - nested_copy(m_data.begin(), t); + base_type::resize(xt::shape(t), default_dynamic_layout()); + L == layout_type::row_major ? nested_copy(m_storage.begin(), t) : nested_copy(this->template begin(), t); } - template - inline pyarray::pyarray(nested_initializer_list_t t) + template + inline pyarray::pyarray(nested_initializer_list_t t) : base_type() { - base_type::resize(xt::shape(t), layout_type::row_major); - nested_copy(m_data.begin(), t); + base_type::resize(xt::shape(t), default_dynamic_layout()); + L == layout_type::row_major ? nested_copy(m_storage.begin(), t) : nested_copy(this->template begin(), t); } - template - inline pyarray::pyarray(pybind11::handle h, pybind11::object::borrowed_t b) + template + inline pyarray::pyarray(pybind11::handle h, pybind11::object::borrowed_t b) : base_type(h, b) { init_from_python(); } - template - inline pyarray::pyarray(pybind11::handle h, pybind11::object::stolen_t s) + template + inline pyarray::pyarray(pybind11::handle h, pybind11::object::stolen_t s) : base_type(h, s) { init_from_python(); } - template - inline pyarray::pyarray(const pybind11::object& o) + template + inline pyarray::pyarray(const pybind11::object& o) : base_type(o) { init_from_python(); @@ -558,8 +329,8 @@ namespace xt * @param shape the shape of the pyarray * @param l the layout of the pyarray */ - template - inline pyarray::pyarray(const shape_type& shape, layout_type l) + template + inline pyarray::pyarray(const shape_type& shape, layout_type l) : base_type() { strides_type strides(shape.size()); @@ -574,14 +345,14 @@ namespace xt * @param value the value of the elements * @param l the layout of the pyarray */ - template - inline pyarray::pyarray(const shape_type& shape, const_reference value, layout_type l) + template + inline pyarray::pyarray(const shape_type& shape, const_reference value, layout_type l) : base_type() { strides_type strides(shape.size()); compute_strides(shape, l, strides); init_array(shape, strides); - std::fill(m_data.begin(), m_data.end(), value); + std::fill(m_storage.begin(), m_storage.end(), value); } /** @@ -591,12 +362,12 @@ namespace xt * @param strides the strides of the pyarray * @param value the value of the elements */ - template - inline pyarray::pyarray(const shape_type& shape, const strides_type& strides, const_reference value) + template + inline pyarray::pyarray(const shape_type& shape, const strides_type& strides, const_reference value) : base_type() { init_array(shape, strides); - std::fill(m_data.begin(), m_data.end(), value); + std::fill(m_storage.begin(), m_storage.end(), value); } /** @@ -604,12 +375,24 @@ namespace xt * @param shape the shape of the pyarray * @param strides the strides of the pyarray */ - template - inline pyarray::pyarray(const shape_type& shape, const strides_type& strides) + template + inline pyarray::pyarray(const shape_type& shape, const strides_type& strides) : base_type() { init_array(shape, strides); } + + /** + * Allocates and returns an pyarray with the specified shape. + * @param shape the shape of the pyarray + */ + template + template + inline pyarray pyarray::from_shape(S&& shape) + { + auto shp = xtl::forward_sequence(shape); + return self_type(shp); + } //@} /** @@ -619,8 +402,8 @@ namespace xt /** * The copy constructor. */ - template - inline pyarray::pyarray(const self_type& rhs) + template + inline pyarray::pyarray(const self_type& rhs) : base_type(), semantic_base(rhs) { auto tmp = pybind11::reinterpret_steal( @@ -633,14 +416,14 @@ namespace xt this->m_ptr = tmp.release().ptr(); init_from_python(); - std::copy(rhs.data().cbegin(), rhs.data().cend(), this->data().begin()); + std::copy(rhs.storage().cbegin(), rhs.storage().cend(), this->storage().begin()); } /** * The assignment operator. */ - template - inline auto pyarray::operator=(const self_type& rhs) -> self_type& + template + inline auto pyarray::operator=(const self_type& rhs) -> self_type& { self_type tmp(rhs); *this = std::move(tmp); @@ -656,15 +439,17 @@ namespace xt /** * The extended copy constructor. */ - template + template template - inline pyarray::pyarray(const xexpression& e) + inline pyarray::pyarray(const xexpression& e) : base_type() { // TODO: prevent intermediary shape allocation - shape_type shape = xtl::forward_sequence(e.derived_cast().shape()); + shape_type shape = xtl::forward_sequence(e.derived_cast().shape()); strides_type strides = xtl::make_sequence(shape.size(), size_type(0)); - compute_strides(shape, layout_type::row_major, strides); + layout_type layout = default_dynamic_layout(); + + compute_strides(shape, layout, strides); init_array(shape, strides); semantic_base::assign(e); } @@ -672,28 +457,28 @@ namespace xt /** * The extended assignment operator. */ - template + template template - inline auto pyarray::operator=(const xexpression& e) -> self_type& + inline auto pyarray::operator=(const xexpression& e) -> self_type& { return semantic_base::operator=(e); } //@} - template - inline auto pyarray::ensure(pybind11::handle h) -> self_type + template + inline auto pyarray::ensure(pybind11::handle h) -> self_type { return base_type::ensure(h); } - template - inline bool pyarray::check_(pybind11::handle h) + template + inline bool pyarray::check_(pybind11::handle h) { return base_type::check_(h); } - template - inline void pyarray::init_array(const shape_type& shape, const strides_type& strides) + template + inline void pyarray::init_array(const shape_type& shape, const strides_type& strides) { strides_type adapted_strides(strides); @@ -705,13 +490,15 @@ namespace xt { flags |= NPY_ARRAY_WRITEABLE; } - int type_num = detail::numpy_traits::type_num; + + auto dtype = pybind11::detail::npy_format_descriptor::dtype(); npy_intp* shape_data = reinterpret_cast(const_cast(shape.data())); npy_intp* strides_data = reinterpret_cast(adapted_strides.data()); + auto tmp = pybind11::reinterpret_steal( - PyArray_New(&PyArray_Type, static_cast(shape.size()), shape_data, type_num, strides_data, - nullptr, static_cast(sizeof(T)), flags, nullptr)); + PyArray_NewFromDescr(&PyArray_Type, (PyArray_Descr*) dtype.release().ptr(), static_cast(shape.size()), shape_data, strides_data, + nullptr, flags, nullptr)); if (!tmp) { @@ -722,32 +509,44 @@ namespace xt init_from_python(); } - template - inline void pyarray::init_from_python() + template + inline void pyarray::init_from_python() { + if (!static_cast(*this)) + { + return; + } + m_shape = inner_shape_type(reinterpret_cast(PyArray_SHAPE(this->python_array())), static_cast(PyArray_NDIM(this->python_array()))); - m_strides = inner_strides_type(reinterpret_cast(PyArray_STRIDES(this->python_array())), - static_cast(PyArray_NDIM(this->python_array()))); + m_strides = inner_strides_type(reinterpret_cast(PyArray_STRIDES(this->python_array())), + static_cast(PyArray_NDIM(this->python_array())), + reinterpret_cast(PyArray_SHAPE(this->python_array()))); + + if (L != layout_type::dynamic && !do_strides_match(m_shape, m_strides, L, 1)) + { + throw std::runtime_error("NumPy: passing container with bad strides for layout (is it a view?)."); + } + m_backstrides = backstrides_type(*this); - m_data = container_type(reinterpret_cast(PyArray_DATA(this->python_array())), - this->get_min_stride() * static_cast(PyArray_SIZE(this->python_array()))); + m_storage = storage_type(reinterpret_cast(PyArray_DATA(this->python_array())), + this->get_buffer_size()); } - template - inline auto pyarray::shape_impl() const noexcept -> const inner_shape_type& + template + inline auto pyarray::shape_impl() const noexcept -> const inner_shape_type& { return m_shape; } - template - inline auto pyarray::strides_impl() const noexcept -> const inner_strides_type& + template + inline auto pyarray::strides_impl() const noexcept -> const inner_strides_type& { return m_strides; } - template - inline auto pyarray::backstrides_impl() const noexcept -> const inner_backstrides_type& + template + inline auto pyarray::backstrides_impl() const noexcept -> const inner_backstrides_type& { // m_backstrides wraps the numpy array backstrides, which is a raw pointer. // The address of the raw pointer stored in the wrapper would be invalidated when the pyarray is copied. @@ -756,16 +555,22 @@ namespace xt return m_backstrides; } - template - inline auto pyarray::data_impl() noexcept -> container_type& + template + inline auto pyarray::storage_impl() noexcept -> storage_type& + { + return m_storage; + } + + template + inline auto pyarray::storage_impl() const noexcept -> const storage_type& { - return m_data; + return m_storage; } - template - inline auto pyarray::data_impl() const noexcept -> const container_type& + template + layout_type pyarray::default_dynamic_layout() { - return m_data; + return L == layout_type::dynamic ? layout_type::row_major : L; } } diff --git a/include/xtensor-python/pyarray_backstrides.hpp b/include/xtensor-python/pyarray_backstrides.hpp new file mode 100644 index 0000000..4dfdab9 --- /dev/null +++ b/include/xtensor-python/pyarray_backstrides.hpp @@ -0,0 +1,378 @@ +/*************************************************************************** +* Copyright (c) Wolf Vollprecht, Johan Mabille and Sylvain Corlay * +* Copyright (c) QuantStack * +* * +* Distributed under the terms of the BSD 3-Clause License. * +* * +* The full license is in the file LICENSE, distributed with this software. * +****************************************************************************/ + +#ifndef PY_ARRAY_BACKSTRIDES_HPP +#define PY_ARRAY_BACKSTRIDES_HPP + +#include +#include + +namespace xt +{ + + /************************** + * pybackstrides_iterator * + **************************/ + + template + class pybackstrides_iterator + { + public: + + using self_type = pybackstrides_iterator; + + using value_type = typename B::value_type; + using pointer = const value_type*; + using reference = value_type; + using difference_type = std::ptrdiff_t; + using iterator_category = std::random_access_iterator_tag; + + pybackstrides_iterator(const B* b, std::size_t offset); + + reference operator*() const; + pointer operator->() const; + + reference operator[](difference_type n) const; + + self_type& operator++(); + self_type& operator--(); + + self_type operator++(int); + self_type operator--(int); + + self_type& operator+=(difference_type n); + self_type& operator-=(difference_type n); + + self_type operator+(difference_type n) const; + self_type operator-(difference_type n) const; + self_type operator-(const self_type& rhs) const; + + std::size_t offset() const; + + private: + + const B* p_b; + std::size_t m_offset; + }; + + template + inline bool operator==(const pybackstrides_iterator& lhs, + const pybackstrides_iterator& rhs); + + template + inline bool operator!=(const pybackstrides_iterator& lhs, + const pybackstrides_iterator& rhs); + + template + inline bool operator<(const pybackstrides_iterator& lhs, + const pybackstrides_iterator& rhs); + + template + inline bool operator<=(const pybackstrides_iterator& lhs, + const pybackstrides_iterator& rhs); + + template + inline bool operator>(const pybackstrides_iterator& lhs, + const pybackstrides_iterator& rhs); + + template + inline bool operator>=(const pybackstrides_iterator& lhs, + const pybackstrides_iterator& rhs); + + /*********************** + * pyarray_backstrides * + ***********************/ + + template + class pyarray_backstrides + { + public: + + using self_type = pyarray_backstrides; + using array_type = A; + using value_type = typename array_type::size_type; + using const_reference = value_type; + using reference = const_reference; + using const_pointer = const value_type*; + using pointer = const_pointer; + using size_type = typename array_type::size_type; + using difference_type = typename array_type::difference_type; + + using const_iterator = pybackstrides_iterator; + using iterator = const_iterator; + using reverse_iterator = std::reverse_iterator; + using const_reverse_iterator = std::reverse_iterator; + + pyarray_backstrides() = default; + pyarray_backstrides(const array_type& a); + + bool empty() const; + size_type size() const; + + value_type operator[](size_type i) const; + + const_reference front() const; + const_reference back() const; + + const_iterator begin() const; + const_iterator end() const; + const_iterator cbegin() const; + const_iterator cend() const; + + const_reverse_iterator rbegin() const; + const_reverse_iterator rend() const; + const_reverse_iterator crbegin() const; + const_reverse_iterator crend() const; + + private: + + const array_type* p_a; + }; + + /***************************************** + * pybackstrides_iterator implementation * + *****************************************/ + + template + inline pybackstrides_iterator::pybackstrides_iterator(const B* b, std::size_t offset) + : p_b(b), m_offset(offset) + { + } + + template + inline auto pybackstrides_iterator::operator*() const -> reference + { + return p_b->operator[](m_offset); + } + + template + inline auto pybackstrides_iterator::operator->() const -> pointer + { + // Returning the address of a temporary + value_type res = p_b->operator[](m_offset); + return &res; + } + + template + inline auto pybackstrides_iterator::operator[](difference_type n) const -> reference + { + return p_b->operator[](m_offset + n); + } + + template + inline auto pybackstrides_iterator::operator++() -> self_type& + { + ++m_offset; + return *this; + } + + template + inline auto pybackstrides_iterator::operator--() -> self_type& + { + --m_offset; + return *this; + } + + template + inline auto pybackstrides_iterator::operator++(int )-> self_type + { + self_type tmp(*this); + ++m_offset; + return tmp; + } + + template + inline auto pybackstrides_iterator::operator--(int) -> self_type + { + self_type tmp(*this); + --m_offset; + return tmp; + } + + template + inline auto pybackstrides_iterator::operator+=(difference_type n) -> self_type& + { + m_offset += n; + return *this; + } + + template + inline auto pybackstrides_iterator::operator-=(difference_type n) -> self_type& + { + m_offset -= n; + return *this; + } + + template + inline auto pybackstrides_iterator::operator+(difference_type n) const -> self_type + { + return self_type(p_b, m_offset + n); + } + + template + inline auto pybackstrides_iterator::operator-(difference_type n) const -> self_type + { + return self_type(p_b, m_offset - n); + } + + template + inline auto pybackstrides_iterator::operator-(const self_type& rhs) const -> self_type + { + self_type tmp(*this); + tmp -= (m_offset - rhs.m_offset); + return tmp; + } + + template + inline std::size_t pybackstrides_iterator::offset() const + { + return m_offset; + } + + template + inline bool operator==(const pybackstrides_iterator& lhs, + const pybackstrides_iterator& rhs) + { + return lhs.offset() == rhs.offset(); + } + + template + inline bool operator!=(const pybackstrides_iterator& lhs, + const pybackstrides_iterator& rhs) + { + return !(lhs == rhs); + } + + template + inline bool operator<(const pybackstrides_iterator& lhs, + const pybackstrides_iterator& rhs) + { + return lhs.offset() < rhs.offset(); + } + + template + inline bool operator<=(const pybackstrides_iterator& lhs, + const pybackstrides_iterator& rhs) + { + return (lhs < rhs) || (lhs == rhs); + } + + template + inline bool operator>(const pybackstrides_iterator& lhs, + const pybackstrides_iterator& rhs) + { + return !(lhs <= rhs); + } + + template + inline bool operator>=(const pybackstrides_iterator& lhs, + const pybackstrides_iterator& rhs) + { + return !(lhs < rhs); + } + + /************************************** + * pyarray_backstrides implementation * + **************************************/ + + template + inline pyarray_backstrides::pyarray_backstrides(const array_type& a) + : p_a(&a) + { + } + + template + inline bool pyarray_backstrides::empty() const + { + return p_a->dimension() == 0; + } + + template + inline auto pyarray_backstrides::size() const -> size_type + { + return p_a->dimension(); + } + + template + inline auto pyarray_backstrides::operator[](size_type i) const -> value_type + { + value_type sh = p_a->shape()[i]; + value_type res = sh == 1 ? 0 : (sh - 1) * p_a->strides()[i]; + return res; + } + + template + inline auto pyarray_backstrides::front() const -> const_reference + { + value_type sh = p_a->shape()[0]; + value_type res = sh == 1 ? 0 : (sh - 1) * p_a->strides()[0]; + return res; + } + + template + inline auto pyarray_backstrides::back() const -> const_reference + { + auto index = p_a->size() - 1; + value_type sh = p_a->shape()[index]; + value_type res = sh == 1 ? 0 : (sh - 1) * p_a->strides()[index]; + return res; + } + + template + inline auto pyarray_backstrides::begin() const -> const_iterator + { + return cbegin(); + } + + template + inline auto pyarray_backstrides::end() const -> const_iterator + { + return cend(); + } + + template + inline auto pyarray_backstrides::cbegin() const -> const_iterator + { + return const_iterator(this, 0); + } + + template + inline auto pyarray_backstrides::cend() const -> const_iterator + { + return const_iterator(this, size()); + } + + template + inline auto pyarray_backstrides::rbegin() const -> const_reverse_iterator + { + return crbegin(); + } + + template + inline auto pyarray_backstrides::rend() const -> const_reverse_iterator + { + return crend(); + } + + template + inline auto pyarray_backstrides::crbegin() const -> const_reverse_iterator + { + return const_reverse_iterator(end()); + } + + template + inline auto pyarray_backstrides::crend() const -> const_reverse_iterator + { + return const_reverse_iterator(begin()); + } + + +} + +#endif diff --git a/include/xtensor-python/pycontainer.hpp b/include/xtensor-python/pycontainer.hpp index bcb5fee..f01c390 100644 --- a/include/xtensor-python/pycontainer.hpp +++ b/include/xtensor-python/pycontainer.hpp @@ -1,5 +1,6 @@ /*************************************************************************** -* Copyright (c) 2016, Johan Mabille and Sylvain Corlay * +* Copyright (c) Wolf Vollprecht, Johan Mabille and Sylvain Corlay * +* Copyright (c) QuantStack * * * * Distributed under the terms of the BSD 3-Clause License. * * * @@ -12,9 +13,11 @@ #include #include #include +#include #include "pybind11/complex.h" #include "pybind11/pybind11.h" +#include "pybind11/numpy.h" #ifndef FORCE_IMPORT_ARRAY #define NO_IMPORT_ARRAY @@ -29,7 +32,7 @@ #undef copysign #include -#include "xtensor/xcontainer.hpp" +#include "xtensor/containers/xcontainer.hpp" #include "xtl/xsequence.hpp" @@ -58,14 +61,14 @@ namespace xt using base_type = xcontainer; using inner_types = xcontainer_inner_types; - using container_type = typename inner_types::container_type; - using value_type = typename container_type::value_type; - using reference = typename container_type::reference; - using const_reference = typename container_type::const_reference; - using pointer = typename container_type::pointer; - using const_pointer = typename container_type::const_pointer; - using size_type = typename container_type::size_type; - using difference_type = typename container_type::difference_type; + using storage_type = typename inner_types::storage_type; + using value_type = typename storage_type::value_type; + using reference = typename storage_type::reference; + using const_reference = typename storage_type::const_reference; + using pointer = typename storage_type::pointer; + using const_pointer = typename storage_type::const_pointer; + using size_type = typename storage_type::size_type; + using difference_type = typename storage_type::difference_type; using shape_type = typename inner_types::shape_type; using strides_type = typename inner_types::strides_type; @@ -73,7 +76,7 @@ namespace xt using inner_shape_type = typename inner_types::inner_shape_type; using inner_strides_type = typename inner_types::inner_strides_type; - using iterable_base = xiterable; + using iterable_base = xcontainer; using iterator = typename iterable_base::iterator; using const_iterator = typename iterable_base::const_iterator; @@ -81,9 +84,6 @@ namespace xt using stepper = typename iterable_base::stepper; using const_stepper = typename iterable_base::const_stepper; - static constexpr layout_type static_layout = layout_type::dynamic; - static constexpr bool contiguous_layout = false; - template void resize(const S& shape); template @@ -92,9 +92,10 @@ namespace xt void resize(const S& shape, const strides_type& strides); template - void reshape(S&& shape, layout_type layout = base_type::static_layout); + auto& reshape(S&& shape, layout_type layout = base_type::static_layout) &; layout_type layout() const; + bool is_contiguous() const noexcept; using base_type::operator(); using base_type::operator[]; @@ -124,13 +125,33 @@ namespace xt const derived_type& derived_cast() const; PyArrayObject* python_array() const; - size_type get_min_stride() const; + size_type get_buffer_size() const; + + private: + +#if (PYBIND11_VERSION_MAJOR == 2 && PYBIND11_VERSION_MINOR >= 3) || PYBIND11_VERSION_MAJOR >= 3 + // Prevent ambiguous overload resolution for operators defined for + // both xt::xcontainer and pybind11::object. + using pybind11::object::operator~; + using pybind11::object::operator+; + using pybind11::object::operator-; + using pybind11::object::operator*; + using pybind11::object::operator/; + using pybind11::object::operator|; + using pybind11::object::operator&; + using pybind11::object::operator^; + using pybind11::object::operator<<; + using pybind11::object::operator>>; +#endif }; namespace detail { + template + struct numpy_traits; + template - struct numpy_traits + struct numpy_traits::value>> { private: @@ -184,6 +205,46 @@ namespace xt { return numpy_enum_adjuster::pyarray_type(obj); } + + template + void default_initialize_impl(T& /*storage*/, std::false_type) + { + } + + template + void default_initialize_impl(T& storage, std::true_type) + { + using value_type = typename T::value_type; + storage[0] = value_type{}; + } + + template + void default_initialize(T& storage) + { + using value_type = typename T::value_type; + default_initialize_impl(storage, std::is_copy_assignable()); + } + + template + bool check_array_type(const pybind11::handle& src, std::true_type) + { + int type_num = xt::detail::numpy_traits::type_num; + return xt::detail::pyarray_type(reinterpret_cast(src.ptr())) == type_num; + } + + template + bool check_array_type(const pybind11::handle& src, std::false_type) + { + return PyArray_EquivTypes((PyArray_Descr*) pybind11::detail::array_proxy(src.ptr())->descr, + (PyArray_Descr*) pybind11::dtype::of().ptr()); + } + + template + bool check_array(const pybind11::handle& src) + { + using is_arithmetic_type = std::integral_constant::value)>; + return PyArray_Check(src.ptr()) && check_array_type(src, is_arithmetic_type{}); + } } /****************************** @@ -232,9 +293,7 @@ namespace xt template inline bool pycontainer::check_(pybind11::handle h) { - int type_num = detail::numpy_traits::type_num; - return PyArray_Check(h.ptr()) && - PyArray_EquivTypenums(PyArray_TYPE(reinterpret_cast(h.ptr())), type_num); + return detail::check_array(h); } template @@ -244,8 +303,9 @@ namespace xt { return nullptr; } - int type_num = detail::numpy_traits::type_num; - auto res = PyArray_FromAny(ptr, PyArray_DescrFromType(type_num), 0, 0, + + auto dtype = pybind11::detail::npy_format_descriptor::dtype(); + auto res = PyArray_FromAny(ptr, (PyArray_Descr *) dtype.release().ptr(), 0, 0, NPY_ARRAY_ENSUREARRAY | NPY_ARRAY_FORCECAST, nullptr); return res; } @@ -257,10 +317,15 @@ namespace xt } template - inline auto pycontainer::get_min_stride() const -> size_type + inline auto pycontainer::get_buffer_size() const -> size_type { const size_type& (*min)(const size_type&, const size_type&) = std::min; - return std::max(size_type(1), std::accumulate(this->strides().cbegin(), this->strides().cend(), std::numeric_limits::max(), min)); + size_type min_stride = this->strides().empty() ? size_type(1) : + std::max(size_type(1), std::accumulate(this->strides().cbegin(), + this->strides().cend(), + std::numeric_limits::max(), + min)); + return min_stride * static_cast(PyArray_SIZE(this->python_array())); } template @@ -275,6 +340,33 @@ namespace xt return *static_cast(this); } + namespace detail + { + template + struct check_dims + { + static bool run(std::size_t) + { + return true; + } + }; + + template + struct check_dims> + { + static bool run(std::size_t new_dim) + { + if(new_dim != N) + { + std::ostringstream err_msg; + err_msg << "Invalid conversion to pycontainer, expecting a container of dimension " + << N << ", got a container of dimension " << new_dim << "."; + throw std::runtime_error(err_msg.str()); + } + return new_dim == N; + } + }; + } /** * resizes the container. @@ -313,23 +405,21 @@ namespace xt template inline void pycontainer::resize(const S& shape, const strides_type& strides) { - derived_type tmp(xtl::forward_sequence(shape), strides); + detail::check_dims::run(shape.size()); + derived_type tmp(xtl::forward_sequence(shape), strides); *static_cast(this) = std::move(tmp); } template template - inline void pycontainer::reshape(S&& shape, layout_type layout) + inline auto& pycontainer::reshape(S&& shape, layout_type layout) & { if (compute_size(shape) != this->size()) { - throw std::runtime_error("Cannot reshape with incorrect number of elements."); - } - - if (layout == layout_type::dynamic || layout == layout_type::any) - { - layout = DEFAULT_LAYOUT; + throw std::runtime_error("Cannot reshape with incorrect number of elements (" + std::to_string(this->size()) + " vs " + std::to_string(compute_size(shape)) + ")"); } + detail::check_dims::run(shape.size()); + layout = default_assignable_layout(layout); NPY_ORDER npy_layout; if (layout == layout_type::row_major) @@ -345,12 +435,14 @@ namespace xt throw std::runtime_error("Cannot reshape with unknown layout_type."); } - PyArray_Dims dims = {reinterpret_cast(shape.data()), static_cast(shape.size())}; + using shape_ptr = typename std::decay_t::pointer; + PyArray_Dims dims = {reinterpret_cast(const_cast(shape.data())), static_cast(shape.size())}; auto new_ptr = PyArray_Newshape((PyArrayObject*) this->ptr(), &dims, npy_layout); auto old_ptr = this->ptr(); this->ptr() = new_ptr; Py_XDECREF(old_ptr); this->derived_cast().init_from_python(); + return *this; } /** @@ -361,11 +453,42 @@ namespace xt inline layout_type pycontainer::layout() const { if (PyArray_CHKFLAGS(python_array(), NPY_ARRAY_C_CONTIGUOUS)) + { return layout_type::row_major; + } else if (PyArray_CHKFLAGS(python_array(), NPY_ARRAY_F_CONTIGUOUS)) + { return layout_type::column_major; + } else + { return layout_type::dynamic; + } + } + + /** + * Return whether or not the container uses contiguous buffer + * @return Boolean for contiguous buffer + */ + template + inline bool pycontainer::is_contiguous() const noexcept + { + if (this->strides().size() == 0) + { + return true; + } + else if (PyArray_CHKFLAGS(python_array(), NPY_ARRAY_C_CONTIGUOUS)) + { + return 1 == this->strides().back(); + } + else if (PyArray_CHKFLAGS(python_array(), NPY_ARRAY_F_CONTIGUOUS)) + { + return 1 == this->strides().front(); + } + else + { + return false; + } } /** @@ -381,6 +504,20 @@ namespace xt } #endif } + +#if defined(__GNUC__) && !defined(__clang__) + namespace workaround + { + // Fixes "undefined symbol" issues + inline void long_long_allocator() + { + std::allocator a; + std::allocator b; + std::allocator c; + std::allocator> d; + } + } +#endif } #endif diff --git a/include/xtensor-python/pynative_casters.hpp b/include/xtensor-python/pynative_casters.hpp new file mode 100644 index 0000000..aa19604 --- /dev/null +++ b/include/xtensor-python/pynative_casters.hpp @@ -0,0 +1,57 @@ +/*************************************************************************** +* Copyright (c) Wolf Vollprecht, Johan Mabille and Sylvain Corlay * +* Copyright (c) QuantStack * +* * +* Distributed under the terms of the BSD 3-Clause License. * +* * +* The full license is in the file LICENSE, distributed with this software. * +****************************************************************************/ + +#ifndef PYNATIVE_CASTERS_HPP +#define PYNATIVE_CASTERS_HPP + +#include "xtensor_type_caster_base.hpp" + +namespace pybind11 +{ + namespace detail + { + // Type caster for casting xarray to ndarray + template + struct type_caster> : xtensor_type_caster_base> + { + }; + + // Type caster for casting xt::xtensor to ndarray + template + struct type_caster> : xtensor_type_caster_base> + { + }; + + // Type caster for casting xt::xtensor_fixed to ndarray + template + struct type_caster> : xtensor_type_caster_base> + { + }; + + // Type caster for casting xt::xstrided_view to ndarray + template + struct type_caster> : xtensor_type_caster_base> + { + }; + + // Type caster for casting xt::xarray_adaptor to ndarray + template + struct type_caster> : xtensor_type_caster_base> + { + }; + + // Type caster for casting xt::xtensor_adaptor to ndarray + template + struct type_caster> : xtensor_type_caster_base> + { + }; + } +} + +#endif diff --git a/include/xtensor-python/pystrides_adaptor.hpp b/include/xtensor-python/pystrides_adaptor.hpp index 8a48c00..bde3923 100644 --- a/include/xtensor-python/pystrides_adaptor.hpp +++ b/include/xtensor-python/pystrides_adaptor.hpp @@ -1,5 +1,6 @@ /*************************************************************************** -* Copyright (c) 2016, Johan Mabille and Sylvain Corlay * +* Copyright (c) Wolf Vollprecht, Johan Mabille and Sylvain Corlay * +* Copyright (c) QuantStack * * * * Distributed under the terms of the BSD 3-Clause License. * * * @@ -27,17 +28,23 @@ namespace xt { public: - using value_type = std::size_t; + using value_type = std::ptrdiff_t; using const_reference = value_type; + using reference = const_reference; using const_pointer = const value_type*; + using pointer = const_pointer; using size_type = std::size_t; using difference_type = std::ptrdiff_t; using const_iterator = pystrides_iterator; + using iterator = const_iterator; using const_reverse_iterator = std::reverse_iterator; + using reverse_iterator = const_reverse_iterator; + + using shape_type = size_t*; pystrides_adaptor() = default; - pystrides_adaptor(const_pointer data, size_type size); + pystrides_adaptor(const_pointer data, size_type size, shape_type shape); bool empty() const noexcept; size_type size() const noexcept; @@ -61,6 +68,7 @@ namespace xt const_pointer p_data; size_type m_size; + shape_type p_shape; }; /********************************** @@ -79,21 +87,25 @@ namespace xt using reference = typename pystrides_adaptor::const_reference; using difference_type = typename pystrides_adaptor::difference_type; using iterator_category = std::random_access_iterator_tag; + using shape_pointer = typename pystrides_adaptor::shape_type; + + pystrides_iterator() = default; - inline pystrides_iterator(pointer current) + inline pystrides_iterator(pointer current, shape_pointer shape) : p_current(current) + , p_shape(shape) { } inline reference operator*() const { - return *p_current / N; + return *p_shape == size_t(1) ? 0 : *p_current / N; } inline pointer operator->() const { // Returning the address of a temporary - value_type res = *p_current / N; + value_type res = this->operator*(); return &res; } @@ -105,12 +117,14 @@ namespace xt inline self_type& operator++() { ++p_current; + ++p_shape; return *this; } inline self_type& operator--() { --p_current; + --p_shape; return *this; } @@ -118,6 +132,7 @@ namespace xt { self_type tmp(*this); ++p_current; + ++p_shape; return tmp; } @@ -125,29 +140,32 @@ namespace xt { self_type tmp(*this); --p_current; + --p_shape; return tmp; } inline self_type& operator+=(difference_type n) { p_current += n; + p_shape += n; return *this; } inline self_type& operator-=(difference_type n) { p_current -= n; + p_shape -= n; return *this; } inline self_type operator+(difference_type n) const { - return self_type(p_current + n); + return self_type(p_current + n, p_shape + n); } inline self_type operator-(difference_type n) const { - return self_type(p_current - n); + return self_type(p_current - n, p_shape - n); } inline difference_type operator-(const self_type& rhs) const @@ -161,6 +179,7 @@ namespace xt private: pointer p_current; + shape_pointer p_shape; }; template @@ -210,8 +229,8 @@ namespace xt ************************************/ template - inline pystrides_adaptor::pystrides_adaptor(const_pointer data, size_type size) - : p_data(data), m_size(size) + inline pystrides_adaptor::pystrides_adaptor(const_pointer data, size_type size, shape_type shape) + : p_data(data), m_size(size), p_shape(shape) { } @@ -230,19 +249,19 @@ namespace xt template inline auto pystrides_adaptor::operator[](size_type i) const -> const_reference { - return p_data[i] / N; + return p_shape[i] == size_t(1) ? 0 : p_data[i] / N; } template inline auto pystrides_adaptor::front() const -> const_reference { - return p_data[0] / N; + return this->operator[](0); } template inline auto pystrides_adaptor::back() const -> const_reference { - return p_data[m_size - 1] / N; + return this->operator[](m_size - 1); } template @@ -260,13 +279,13 @@ namespace xt template inline auto pystrides_adaptor::cbegin() const -> const_iterator { - return const_iterator(p_data); + return const_iterator(p_data, p_shape); } template inline auto pystrides_adaptor::cend() const -> const_iterator { - return const_iterator(p_data + m_size); + return const_iterator(p_data + m_size, p_shape + m_size); } template diff --git a/include/xtensor-python/pytensor.hpp b/include/xtensor-python/pytensor.hpp index 7bad5d7..e3746cc 100644 --- a/include/xtensor-python/pytensor.hpp +++ b/include/xtensor-python/pytensor.hpp @@ -1,5 +1,6 @@ /*************************************************************************** -* Copyright (c) 2016, Johan Mabille and Sylvain Corlay * +* Copyright (c) Wolf Vollprecht, Johan Mabille and Sylvain Corlay * +* Copyright (c) QuantStack * * * * Distributed under the terms of the BSD 3-Clause License. * * * @@ -13,18 +14,20 @@ #include #include -#include "xtensor/xbuffer_adaptor.hpp" -#include "xtensor/xiterator.hpp" -#include "xtensor/xsemantic.hpp" -#include "xtensor/xutils.hpp" +#include "xtensor/containers/xbuffer_adaptor.hpp" +#include "xtensor/core/xiterator.hpp" +#include "xtensor/core/xsemantic.hpp" +#include "xtensor/utils/xutils.hpp" #include "pycontainer.hpp" #include "pystrides_adaptor.hpp" +#include "pynative_casters.hpp" #include "xtensor_type_caster_base.hpp" +#include "xtensor_python_config.hpp" namespace xt { - template + template class pytensor; } @@ -32,36 +35,40 @@ namespace pybind11 { namespace detail { - template - struct handle_type_name> +#ifdef PYBIND11_DESCR // The macro is removed from pybind11 since 2.3 + template + struct handle_type_name> { static PYBIND11_DESCR name() { - return _("numpy.ndarray[") + make_caster::name() + _("]"); + return _("numpy.ndarray[") + npy_format_descriptor::name() + _("]"); } }; +#endif - template - struct pyobject_caster> + template + struct pyobject_caster> { - using type = xt::pytensor; + using type = xt::pytensor; bool load(handle src, bool convert) { if (!convert) { - if (!PyArray_Check(src.ptr())) - { - return false; - } - int type_num = xt::detail::numpy_traits::type_num; - if(xt::detail::pyarray_type(reinterpret_cast(src.ptr())) != type_num) + if (!xt::detail::check_array(src)) { return false; } } - value = type::ensure(src); + try + { + value = type::ensure(src); + } + catch (const std::runtime_error&) + { + return false; + } return static_cast(value); } @@ -70,14 +77,18 @@ namespace pybind11 return src.inc_ref(); } +#ifdef PYBIND11_DESCR // The macro is removed from pybind11 since 2.3 PYBIND11_TYPE_CASTER(type, handle_type_name::name()); +#else + PYBIND11_TYPE_CASTER(type, _("numpy.ndarray[") + npy_format_descriptor::name + _("]")); +#endif }; // Type caster for casting ndarray to xexpression - template - struct type_caster>> : pyobject_caster> + template + struct type_caster>> : pyobject_caster> { - using Type = xt::xexpression>; + using Type = xt::xexpression>; operator Type&() { @@ -90,35 +101,48 @@ namespace pybind11 } }; - // Type caster for casting xt::xtensor to ndarray - template - struct type_caster> : xtensor_type_caster_base> - { - }; - } + } // namespace detail } namespace xt { + namespace detail { + + template + struct numpy_strides + { + npy_intp value[N]; + }; + + template <> + struct numpy_strides<0> + { + npy_intp* value = nullptr; + }; - template - struct xiterable_inner_types> - : xcontainer_iterable_types> + } // namespace detail + + template + struct xiterable_inner_types> + : xcontainer_iterable_types> { }; - template - struct xcontainer_inner_types> + template + struct xcontainer_inner_types> { - using container_type = xbuffer_adaptor; + using storage_type = xbuffer_adaptor; + using reference = typename storage_type::reference; + using const_reference = typename storage_type::const_reference; + using size_type = typename storage_type::size_type; using shape_type = std::array; using strides_type = shape_type; using backstrides_type = shape_type; using inner_shape_type = shape_type; using inner_strides_type = strides_type; using inner_backstrides_type = backstrides_type; - using temporary_type = pytensor; - static constexpr layout_type layout = layout_type::dynamic; + using temporary_type = pytensor; + static constexpr layout_type layout = L; }; /** @@ -127,7 +151,7 @@ namespace xt * * pytensor is similar to the xtensor container in that it has a static dimensionality. * - * Unlike with the pyarray container, pytensor cannot be reshaped with a different number of dimensions + * Unlike the pyarray container, pytensor cannot be reshaped with a different number of dimensions * and reshapes are not reflected on the Python side. However, pytensor has benefits compared to pyarray * in terms of performances. pytensor shapes are stack-allocated which makes iteration upon pytensor * faster than with pyarray. @@ -135,16 +159,16 @@ namespace xt * @tparam T The type of the element stored in the pyarray. * @sa pyarray */ - template - class pytensor : public pycontainer>, - public xcontainer_semantic> + template + class pytensor : public pycontainer>, + public xcontainer_semantic> { public: - using self_type = pytensor; + using self_type = pytensor; using semantic_base = xcontainer_semantic; using base_type = pycontainer; - using container_type = typename base_type::container_type; + using storage_type = typename base_type::storage_type; using value_type = typename base_type::value_type; using reference = typename base_type::reference; using const_reference = typename base_type::const_reference; @@ -156,6 +180,7 @@ namespace xt using inner_shape_type = typename base_type::inner_shape_type; using inner_strides_type = typename base_type::inner_strides_type; using inner_backstrides_type = typename base_type::inner_backstrides_type; + constexpr static std::size_t rank = N; pytensor(); pytensor(nested_initializer_list_t t); @@ -168,6 +193,9 @@ namespace xt explicit pytensor(const shape_type& shape, const strides_type& strides, const_reference value); explicit pytensor(const shape_type& shape, const strides_type& strides); + template + static pytensor from_shape(S&& shape); + pytensor(const self_type& rhs); self_type& operator=(const self_type& rhs); @@ -186,12 +214,26 @@ namespace xt static self_type ensure(pybind11::handle h); static bool check_(pybind11::handle h); +#if (PYBIND11_VERSION_MAJOR == 2 && PYBIND11_VERSION_MINOR >= 3) || (PYBIND11_VERSION_MAJOR >= 3) + // Prevent ambiguous overload resolution for operators defined for + // both xt::xcontainer_semantic and pybind11::object. + using semantic_base::operator+=; + using semantic_base::operator-=; + using semantic_base::operator*=; + using semantic_base::operator/=; + using semantic_base::operator|=; + using semantic_base::operator&=; + using semantic_base::operator^=; + // using semantic_base::operator<<=; + // using semantic_base::operator>>=; +#endif + private: inner_shape_type m_shape; inner_strides_type m_strides; inner_backstrides_type m_backstrides; - container_type m_data; + storage_type m_storage; void init_tensor(const shape_type& shape, const strides_type& strides); void init_from_python(); @@ -203,11 +245,11 @@ namespace xt inner_backstrides_type& backstrides_impl() noexcept; const inner_backstrides_type& backstrides_impl() const noexcept; - container_type& data_impl() noexcept; - const container_type& data_impl() const noexcept; + storage_type& storage_impl() noexcept; + const storage_type& storage_impl() const noexcept; - friend class xcontainer>; - friend class pycontainer>; + friend class xcontainer>; + friend class pycontainer>; }; /*************************** @@ -221,43 +263,43 @@ namespace xt /** * Allocates an uninitialized pytensor that holds 1 element. */ - template - inline pytensor::pytensor() + template + inline pytensor::pytensor() : base_type() { m_shape = xtl::make_sequence(N, size_type(1)); m_strides = xtl::make_sequence(N, size_type(0)); init_tensor(m_shape, m_strides); - m_data[0] = T(); + detail::default_initialize(m_storage); } /** * Allocates a pytensor with a nested initializer list. */ - template - inline pytensor::pytensor(nested_initializer_list_t t) + template + inline pytensor::pytensor(nested_initializer_list_t t) : base_type() { base_type::resize(xt::shape(t), layout_type::row_major); - nested_copy(m_data.begin(), t); + nested_copy(m_storage.begin(), t); } - template - inline pytensor::pytensor(pybind11::handle h, pybind11::object::borrowed_t b) + template + inline pytensor::pytensor(pybind11::handle h, pybind11::object::borrowed_t b) : base_type(h, b) { init_from_python(); } - template - inline pytensor::pytensor(pybind11::handle h, pybind11::object::stolen_t s) + template + inline pytensor::pytensor(pybind11::handle h, pybind11::object::stolen_t s) : base_type(h, s) { init_from_python(); } - template - inline pytensor::pytensor(const pybind11::object& o) + template + inline pytensor::pytensor(const pybind11::object& o) : base_type(o) { init_from_python(); @@ -269,8 +311,8 @@ namespace xt * @param shape the shape of the pytensor * @param l the layout_type of the pytensor */ - template - inline pytensor::pytensor(const shape_type& shape, layout_type l) + template + inline pytensor::pytensor(const shape_type& shape, layout_type l) { compute_strides(shape, l, m_strides); init_tensor(shape, m_strides); @@ -283,14 +325,14 @@ namespace xt * @param value the value of the elements * @param l the layout_type of the pytensor */ - template - inline pytensor::pytensor(const shape_type& shape, + template + inline pytensor::pytensor(const shape_type& shape, const_reference value, layout_type l) { compute_strides(shape, l, m_strides); init_tensor(shape, m_strides); - std::fill(m_data.begin(), m_data.end(), value); + std::fill(m_storage.begin(), m_storage.end(), value); } /** @@ -300,13 +342,13 @@ namespace xt * @param strides the strides of the pytensor * @param value the value of the elements */ - template - inline pytensor::pytensor(const shape_type& shape, + template + inline pytensor::pytensor(const shape_type& shape, const strides_type& strides, const_reference value) { init_tensor(shape, strides); - std::fill(m_data.begin(), m_data.end(), value); + std::fill(m_storage.begin(), m_storage.end(), value); } /** @@ -314,12 +356,25 @@ namespace xt * @param shape the shape of the pytensor * @param strides the strides of the pytensor */ - template - inline pytensor::pytensor(const shape_type& shape, + template + inline pytensor::pytensor(const shape_type& shape, const strides_type& strides) { init_tensor(shape, strides); } + + /** + * Allocates and returns an pytensor with the specified shape. + * @param shape the shape of the pytensor + */ + template + template + inline pytensor pytensor::from_shape(S&& shape) + { + detail::check_dims::run(shape.size()); + auto shp = xtl::forward_sequence(shape); + return self_type(shp); + } //@} /** @@ -329,19 +384,19 @@ namespace xt /** * The copy constructor. */ - template - inline pytensor::pytensor(const self_type& rhs) - : base_type() + template + inline pytensor::pytensor(const self_type& rhs) + : base_type(), semantic_base(rhs) { init_tensor(rhs.shape(), rhs.strides()); - std::copy(rhs.data().cbegin(), rhs.data().cend(), this->data().begin()); + std::copy(rhs.storage().cbegin(), rhs.storage().cend(), this->storage().begin()); } /** * The assignment operator. */ - template - inline auto pytensor::operator=(const self_type& rhs) -> self_type& + template + inline auto pytensor::operator=(const self_type& rhs) -> self_type& { self_type tmp(rhs); *this = std::move(tmp); @@ -356,12 +411,12 @@ namespace xt /** * The extended copy constructor. */ - template + template template - inline pytensor::pytensor(const xexpression& e) + inline pytensor::pytensor(const xexpression& e) : base_type() { - shape_type shape = xtl::forward_sequence(e.derived_cast().shape()); + shape_type shape = xtl::forward_sequence(e.derived_cast().shape()); strides_type strides = xtl::make_sequence(N, size_type(0)); compute_strides(shape, layout_type::row_major, strides); init_tensor(shape, strides); @@ -371,42 +426,43 @@ namespace xt /** * The extended assignment operator. */ - template + template template - inline auto pytensor::operator=(const xexpression& e) -> self_type& + inline auto pytensor::operator=(const xexpression& e) -> self_type& { return semantic_base::operator=(e); } //@} - template - inline auto pytensor::ensure(pybind11::handle h) -> self_type + template + inline auto pytensor::ensure(pybind11::handle h) -> self_type { return base_type::ensure(h); } - template - inline bool pytensor::check_(pybind11::handle h) + template + inline bool pytensor::check_(pybind11::handle h) { return base_type::check_(h); } - template - inline void pytensor::init_tensor(const shape_type& shape, const strides_type& strides) + template + inline void pytensor::init_tensor(const shape_type& shape, const strides_type& strides) { - npy_intp python_strides[N]; - std::transform(strides.begin(), strides.end(), python_strides, + detail::numpy_strides python_strides; + std::transform(strides.begin(), strides.end(), python_strides.value, [](auto v) { return sizeof(T) * v; }); int flags = NPY_ARRAY_ALIGNED; if (!std::is_const::value) { flags |= NPY_ARRAY_WRITEABLE; } - int type_num = detail::numpy_traits::type_num; + auto dtype = pybind11::detail::npy_format_descriptor::dtype(); auto tmp = pybind11::reinterpret_steal( - PyArray_New(&PyArray_Type, N, const_cast(shape.data()), - type_num, python_strides, nullptr, sizeof(T), flags, nullptr)); + PyArray_NewFromDescr(&PyArray_Type, (PyArray_Descr*) dtype.release().ptr(), static_cast(shape.size()), + const_cast(shape.data()), python_strides.value, + nullptr, flags, nullptr)); if (!tmp) { @@ -417,13 +473,18 @@ namespace xt m_shape = shape; m_strides = strides; adapt_strides(m_shape, m_strides, m_backstrides); - m_data = container_type(reinterpret_cast(PyArray_DATA(this->python_array())), - static_cast(PyArray_SIZE(this->python_array()))); + m_storage = storage_type(reinterpret_cast(PyArray_DATA(this->python_array())), + static_cast(PyArray_SIZE(this->python_array()))); } - template - inline void pytensor::init_from_python() + template + inline void pytensor::init_from_python() { + if (!static_cast(*this)) + { + return; + } + if (PyArray_NDIM(this->python_array()) != N) { throw std::runtime_error("NumPy: ndarray has incorrect number of dimensions"); @@ -433,56 +494,62 @@ namespace xt std::transform(PyArray_STRIDES(this->python_array()), PyArray_STRIDES(this->python_array()) + N, m_strides.begin(), [](auto v) { return v / sizeof(T); }); adapt_strides(m_shape, m_strides, m_backstrides); - m_data = container_type(reinterpret_cast(PyArray_DATA(this->python_array())), - this->get_min_stride() * static_cast(PyArray_SIZE(this->python_array()))); + + if (L != layout_type::dynamic && !do_strides_match(m_shape, m_strides, L, 1)) + { + throw std::runtime_error("NumPy: passing container with bad strides for layout (is it a view?)."); + } + + m_storage = storage_type(reinterpret_cast(PyArray_DATA(this->python_array())), + this->get_buffer_size()); } - template - inline auto pytensor::shape_impl() noexcept -> inner_shape_type& + template + inline auto pytensor::shape_impl() noexcept -> inner_shape_type& { return m_shape; } - template - inline auto pytensor::shape_impl() const noexcept -> const inner_shape_type& + template + inline auto pytensor::shape_impl() const noexcept -> const inner_shape_type& { return m_shape; } - template - inline auto pytensor::strides_impl() noexcept -> inner_strides_type& + template + inline auto pytensor::strides_impl() noexcept -> inner_strides_type& { return m_strides; } - template - inline auto pytensor::strides_impl() const noexcept -> const inner_strides_type& + template + inline auto pytensor::strides_impl() const noexcept -> const inner_strides_type& { return m_strides; } - template - inline auto pytensor::backstrides_impl() noexcept -> inner_backstrides_type& + template + inline auto pytensor::backstrides_impl() noexcept -> inner_backstrides_type& { return m_backstrides; } - template - inline auto pytensor::backstrides_impl() const noexcept -> const inner_backstrides_type& + template + inline auto pytensor::backstrides_impl() const noexcept -> const inner_backstrides_type& { return m_backstrides; } - template - inline auto pytensor::data_impl() noexcept -> container_type& + template + inline auto pytensor::storage_impl() noexcept -> storage_type& { - return m_data; + return m_storage; } - template - inline auto pytensor::data_impl() const noexcept -> const container_type& + template + inline auto pytensor::storage_impl() const noexcept -> const storage_type& { - return m_data; + return m_storage; } } diff --git a/include/xtensor-python/pyvectorize.hpp b/include/xtensor-python/pyvectorize.hpp index 75c1473..d96f689 100644 --- a/include/xtensor-python/pyvectorize.hpp +++ b/include/xtensor-python/pyvectorize.hpp @@ -1,5 +1,6 @@ /*************************************************************************** -* Copyright (c) 2016, Johan Mabille and Sylvain Corlay * +* Copyright (c) Wolf Vollprecht, Johan Mabille and Sylvain Corlay * +* Copyright (c) QuantStack * * * * Distributed under the terms of the BSD 3-Clause License. * * * @@ -12,7 +13,7 @@ #include #include "pyarray.hpp" -#include "xtensor/xvectorize.hpp" +#include "xtensor/core/xvectorize.hpp" namespace xt { diff --git a/include/xtensor-python/xtensor_python_config.hpp b/include/xtensor-python/xtensor_python_config.hpp index d2d0839..4bb0882 100644 --- a/include/xtensor-python/xtensor_python_config.hpp +++ b/include/xtensor-python/xtensor_python_config.hpp @@ -1,5 +1,6 @@ /*************************************************************************** -* Copyright (c) 2016, Johan Mabille and Sylvain Corlay * +* Copyright (c) Wolf Vollprecht, Johan Mabille and Sylvain Corlay * +* Copyright (c) QuantStack * * * * Distributed under the terms of the BSD 3-Clause License. * * * @@ -10,7 +11,7 @@ #define XTENSOR_PYTHON_CONFIG_HPP #define XTENSOR_PYTHON_VERSION_MAJOR 0 -#define XTENSOR_PYTHON_VERSION_MINOR 17 +#define XTENSOR_PYTHON_VERSION_MINOR 29 #define XTENSOR_PYTHON_VERSION_PATCH 0 #endif diff --git a/include/xtensor-python/xtensor_type_caster_base.hpp b/include/xtensor-python/xtensor_type_caster_base.hpp index 41e43ef..902235f 100644 --- a/include/xtensor-python/xtensor_type_caster_base.hpp +++ b/include/xtensor-python/xtensor_type_caster_base.hpp @@ -1,21 +1,21 @@ -/* - xtensor-python/xtensor_type_caster.hpp: Transparent conversion for xtensor and xarray - - This code is based on the following code written by Wenzel Jakob - - pybind11/eigen.h: Transparent conversion for dense and sparse Eigen matrices - - Copyright (c) 2016 Wenzel Jakob - - All rights reserved. Use of this source code is governed by a - BSD-style license that can be found in the LICENSE file. -*/ - +/*************************************************************************** +* Copyright (c) Wolf Vollprecht, Johan Mabille and Sylvain Corlay * +* Copyright (c) QuantStack * +* * +* Distributed under the terms of the BSD 3-Clause License. * +* * +* The full license is in the file LICENSE, distributed with this software. * +****************************************************************************/ #ifndef XTENSOR_TYPE_CASTER_HPP #define XTENSOR_TYPE_CASTER_HPP -#include "xtensor/xtensor.hpp" +#include +#include +#include + +#include "xtensor/containers/xtensor.hpp" +#include "xtensor/containers/xfixed.hpp" #include #include @@ -24,20 +24,154 @@ namespace pybind11 { namespace detail { - // Casts an xtensor (or xarray) type to numpy array. If given a base, the numpy array references the src data, - // otherwise it'll make a copy. writeable lets you turn off the writeable flag for the array. + template + struct pybind_array_getter_impl + { + static auto run(handle src) + { + return array_t::ensure(src); + } + }; + + template + struct pybind_array_getter_impl + { + static auto run(handle src) + { + return array_t::ensure(src); + } + }; + + template + struct pybind_array_getter + { + }; + + template + struct pybind_array_getter> + { + static auto run(handle src) + { + return pybind_array_getter_impl::run(src); + } + }; + + template + struct pybind_array_getter> + { + static auto run(handle src) + { + return pybind_array_getter_impl::run(src); + } + }; + + template + struct pybind_array_getter> + { + static auto run(handle src) + { + return pybind_array_getter_impl::run(src); + } + }; + + template + struct pybind_array_getter> + { + static auto run(handle /*src*/) + { + return false; + } + }; + + template + struct pybind_array_getter> + { + static auto run(handle src) + { + auto buf = pybind_array_getter_impl::run(src); + return buf; + } + }; + + template + struct pybind_array_getter> + { + static auto run(handle /*src*/) + { + return false; + } + }; + + + template + struct pybind_array_dim_checker + { + template + static bool run(const B& /*buf*/) + { + return true; + } + }; + + template + struct pybind_array_dim_checker> + { + template + static bool run(const B& buf) + { + return buf.ndim() == N; + } + }; + + template + struct pybind_array_dim_checker> + { + template + static bool run(const B& buf) + { + return buf.ndim() == FSH::size(); + } + }; + + + template + struct pybind_array_shape_checker + { + template + static bool run(const B& /*buf*/) + { + return true; + } + }; + + template + struct pybind_array_shape_checker> + { + template + static bool run(const B& buf) + { + auto shape = FSH(); + return std::equal(shape.begin(), shape.end(), buf.shape()); + } + }; + + // Casts a strided expression type to numpy array.If given a base, + // the numpy array references the src data, otherwise it'll make a copy. + // The writeable attributes lets you specify writeable flag for the array. template - handle xtensor_array_cast(Type const& src, handle base = handle(), bool writeable = true) + handle xtensor_array_cast(const Type& src, handle base = handle(), bool writeable = true) { // TODO: make use of xt::pyarray instead of array. - std::vector python_strides(src.strides().size()); - std::transform(src.strides().begin(), src.strides().end(), python_strides.begin(), - [](auto v) { return sizeof(typename Type::value_type) * v; }); + std::vector python_strides(src.strides().size()); + std::transform(src.strides().begin(), src.strides().end(), + python_strides.begin(), [](auto v) { + return sizeof(typename Type::value_type) * v; + }); - std::vector python_shape(src.shape().size()); + std::vector python_shape(src.shape().size()); std::copy(src.shape().begin(), src.shape().end(), python_shape.begin()); - array a(python_shape, python_strides, src.begin(), base); + array a(python_shape, python_strides, &*(src.begin()), base); if (!writeable) { @@ -47,8 +181,8 @@ namespace pybind11 return a.release(); } - // Takes an lvalue ref to some xtensor (or xarray) type and a (python) base object, creating a numpy array that - // reference the xtensor object's data with `base` as the python-registered base class (if omitted, + // Takes an lvalue ref to some strided expression type and a (python) base object, creating a numpy array that + // reference the expression object's data with `base` as the python-registered base class (if omitted, // the base will be set to None, and lifetime management is up to the caller). The numpy array is // non-writeable if the given type is const. template @@ -57,7 +191,7 @@ namespace pybind11 return xtensor_array_cast(src, parent, !std::is_const::value); } - // Takes a pointer to xtensor (or xarray), builds a capsule around it, then returns a numpy + // Takes a pointer to a strided expression, builds a capsule around it, then returns a numpy // array that references the encapsulated data with a python-side reference to the capsule to tie // its destruction to that of any dependent python objects. Const-ness is determined by whether or // not the CType of the pointer given is const. @@ -68,14 +202,10 @@ namespace pybind11 return xtensor_ref_array(*src, base); } - // Base class of type_caster for xtensor and xarray + // Base class of type_caster for strided expressions template struct xtensor_type_caster_base { - bool load(handle src, bool) - { - return false; - } private: @@ -104,6 +234,41 @@ namespace pybind11 public: + PYBIND11_TYPE_CASTER(Type, _("numpy.ndarray[") + npy_format_descriptor::name + _("]")); + + bool load(handle src, bool convert) + { + using T = typename Type::value_type; + + if (!convert && !array_t::check_(src)) + { + return false; + } + + auto buf = pybind_array_getter::run(src); + + if (!buf) + { + return false; + } + if (!pybind_array_dim_checker::run(buf)) + { + return false; + } + + if (!pybind_array_shape_checker::run(buf)) + { + return false; + } + + std::vector shape(buf.ndim()); + std::copy(buf.shape(), buf.shape() + buf.ndim(), shape.begin()); + value = Type::from_shape(shape); + std::copy(buf.data(), buf.data() + buf.size(), value.data()); + + return true; + } + // Normal returned non-reference, non-const value: static handle cast(Type&& src, return_value_policy /* policy */, handle parent) { @@ -149,14 +314,6 @@ namespace pybind11 { return cast_impl(src, policy, parent); } - - static PYBIND11_DESCR name() - { - return _("xt::xtensor"); - } - - template - using cast_op_type = cast_op_type; }; } } diff --git a/readthedocs.yml b/readthedocs.yml index 02f0e7b..a8188c4 100644 --- a/readthedocs.yml +++ b/readthedocs.yml @@ -1,2 +1,13 @@ +version: 2 + +build: + os: "ubuntu-22.04" + tools: + python: "mambaforge-22.9" + +sphinx: + # Path to Sphinx configuration file + configuration: docs/source/conf.py + conda: - file: docs/environment.yml + environment: docs/environment.yml diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 2f03448..60fcf10 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,12 +1,13 @@ ############################################################################ -# Copyright (c) 2016, Johan Mabille and Sylvain Corlay # +# Copyright (c) Wolf Vollprecht, Johan Mabille and Sylvain Corlay # +# Copyright (c) QuantStack # # # # Distributed under the terms of the BSD 3-Clause License. # # # # The full license is in the file LICENSE, distributed with this software. # ############################################################################ -cmake_minimum_required(VERSION 3.1) +cmake_minimum_required(VERSION 3.29) if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR) project(xtensor-python-test) @@ -28,16 +29,7 @@ include(CheckCXXCompilerFlag) string(TOUPPER "${CMAKE_BUILD_TYPE}" U_CMAKE_BUILD_TYPE) -if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Intel") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native -Wunused-parameter -Wextra -Wreorder -Wconversion") - CHECK_CXX_COMPILER_FLAG("-std=c++14" HAS_CPP14_FLAG) - - if (HAS_CPP14_FLAG) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14") - else() - message(FATAL_ERROR "Unsupported compiler -- xtensor requires C++14 support!") - endif() -endif() +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17") if(MSVC) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc /MP /bigobj") @@ -65,10 +57,12 @@ if (DOWNLOAD_GTEST OR GTEST_SRC_DIR) message(FATAL_ERROR "Build step for googletest failed: ${result}") endif() + set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) + # Add googletest directly to our build. This defines # the gtest and gtest_main targets. add_subdirectory(${CMAKE_CURRENT_BINARY_DIR}/googletest-src - ${CMAKE_CURRENT_BINARY_DIR}/googletest-build) + ${CMAKE_CURRENT_BINARY_DIR}/googletest-build EXCLUDE_FROM_ALL) set(GTEST_INCLUDE_DIRS "${gtest_SOURCE_DIR}/include") set(GTEST_BOTH_LIBRARIES gtest_main gtest) @@ -83,17 +77,18 @@ include_directories(${GTEST_INCLUDE_DIRS}) set(XTENSOR_PYTHON_TESTS main.cpp test_pyarray.cpp + test_pyarray_traits.cpp test_pytensor.cpp test_pyvectorize.cpp + test_sfinae.cpp ) -set(XTENSOR_PYTHON_TARGET test_xtensor_python) -add_executable(${XTENSOR_PYTHON_TARGET} ${XTENSOR_PYTHON_TESTS} ${XTENSOR_PYTHON_HEADERS}) -target_link_libraries(${XTENSOR_PYTHON_TARGET} xtensor-python ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${PYTHON_LIBRARIES}) +add_executable(test_xtensor_python ${XTENSOR_PYTHON_TESTS} ${XTENSOR_PYTHON_HEADERS}) +target_link_libraries(test_xtensor_python xtensor-python ${GTEST_BOTH_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT} ${PYTHON_LIBRARIES}) if(DOWNLOAD_GTEST OR GTEST_SRC_DIR) - add_dependencies(${XTENSOR_PYTHON_TARGET} gtest_main) + add_dependencies(test_xtensor_python gtest_main) endif() -add_custom_target(xtest COMMAND ./test_xtensor_python DEPENDS ${XTENSOR_PYTHON_TARGET}) +add_custom_target(xtest COMMAND ./test_xtensor_python DEPENDS test_xtensor_python) diff --git a/test/copyGTest.cmake.in b/test/copyGTest.cmake.in index db58282..50821d0 100644 --- a/test/copyGTest.cmake.in +++ b/test/copyGTest.cmake.in @@ -1,12 +1,13 @@ ############################################################################ -# Copyright (c) 2016, Johan Mabille and Sylvain Corlay # +# Copyright (c) Wolf Vollprecht, Johan Mabille and Sylvain Corlay # +# Copyright (c) QuantStack # # # # Distributed under the terms of the BSD 3-Clause License. # # # # The full license is in the file LICENSE, distributed with this software. # ############################################################################ -cmake_minimum_required(VERSION 2.8.2) +cmake_minimum_required(VERSION 3.29) project(googletest-download NONE) diff --git a/test/downloadGTest.cmake.in b/test/downloadGTest.cmake.in index 69a8bad..6bb5fad 100644 --- a/test/downloadGTest.cmake.in +++ b/test/downloadGTest.cmake.in @@ -1,19 +1,20 @@ ############################################################################ -# Copyright (c) 2016, Johan Mabille and Sylvain Corlay # +# Copyright (c) Wolf Vollprecht, Johan Mabille and Sylvain Corlay # +# Copyright (c) QuantStack # # # # Distributed under the terms of the BSD 3-Clause License. # # # # The full license is in the file LICENSE, distributed with this software. # ############################################################################ -cmake_minimum_required(VERSION 2.8.2) +cmake_minimum_required(VERSION 3.29) project(googletest-download NONE) include(ExternalProject) ExternalProject_Add(googletest GIT_REPOSITORY https://github.com/google/googletest.git - GIT_TAG release-1.8.0 + GIT_TAG v1.16.0 SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/googletest-src" BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/googletest-build" CONFIGURE_COMMAND "" diff --git a/test/main.cpp b/test/main.cpp index 87f5685..d3cc6c4 100644 --- a/test/main.cpp +++ b/test/main.cpp @@ -1,35 +1,35 @@ /*************************************************************************** -* Copyright (c) 2016, Johan Mabille and Sylvain Corlay * +* Copyright (c) Wolf Vollprecht, Johan Mabille and Sylvain Corlay * +* Copyright (c) QuantStack * * * * Distributed under the terms of the BSD 3-Clause License. * * * * The full license is in the file LICENSE, distributed with this software. * ****************************************************************************/ -// Required to avoid the error "std does not have memeber copysign" +// Required to avoid the error "std does not have member copysign" #include -#include -#include "pybind11/numpy.h" +#include "gtest/gtest.h" + +#include #define FORCE_IMPORT_ARRAY #include "xtensor-python/pyarray.hpp" -#include "gtest/gtest.h" -#include +namespace py = pybind11; int main(int argc, char* argv[]) { - // Initialize all the things (google-test and Python interpreter) - Py_Initialize(); + // Initialize all the things (Python, numpy, gtest) + py::scoped_interpreter guard{}; xt::import_numpy(); ::testing::InitGoogleTest(&argc, argv); // Run test suite int ret = RUN_ALL_TESTS(); - // Closure of the Python interpreter - Py_Finalize(); + // Return test results return ret; } diff --git a/test/test_common.hpp b/test/test_common.hpp index 5c7fc00..7115764 100644 --- a/test/test_common.hpp +++ b/test/test_common.hpp @@ -1,5 +1,6 @@ /*************************************************************************** -* Copyright (c) 2016, Johan Mabille, Sylvain Corlay and Wolf Vollprecht * +* Copyright (c) Wolf Vollprecht, Johan Mabille and Sylvain Corlay * +* Copyright (c) QuantStack * * * * Distributed under the terms of the BSD 3-Clause License. * * * @@ -9,8 +10,8 @@ #ifndef TEST_COMMON_HPP #define TEST_COMMON_HPP -#include "xtensor/xlayout.hpp" -#include "xtensor/xstrided_view.hpp" +#include "xtensor/core/xlayout.hpp" +#include "xtensor/misc/xmanipulation.hpp" #include "xtl/xsequence.hpp" @@ -34,7 +35,7 @@ namespace xt using vector_type = uvector; using size_type = typename C::value_type; using shape_type = C; - using strides_type = C; + using strides_type = get_strides_t; using assigner_type = std::vector>; @@ -159,7 +160,10 @@ namespace xt { EXPECT_TRUE(std::equal(vec.shape().cbegin(), vec.shape().cend(), result.shape().cbegin())); EXPECT_TRUE(std::equal(vec.strides().cbegin(), vec.strides().cend(), result.strides().cbegin())); +// TODO: check why this does not build on modern MSVC compilers +#ifndef WIN32 EXPECT_TRUE(std::equal(vec.backstrides().cbegin(), vec.backstrides().cend(), result.backstrides().cbegin())); +#endif EXPECT_EQ(vec.size(), result.size()); if (compare_layout) { @@ -234,7 +238,7 @@ namespace xt vec.resize(rm.shape(), layout_type::row_major); assign_array(vec, rm.m_assigner); - EXPECT_TRUE(std::equal(vec.data().cbegin(), vec.data().cend(), rm.m_data.cbegin())); + EXPECT_TRUE(std::equal(vec.storage().cbegin(), vec.storage().cend(), rm.m_data.cbegin())); auto vec_copy = vec; @@ -242,7 +246,7 @@ namespace xt auto vt = transpose(vec); std::reverse(shape_new.begin(), shape_new.end()); EXPECT_EQ(vt.shape(), shape_new); - EXPECT_TRUE(std::equal(vt.data().cbegin(), vt.data().cend(), rm.m_data.cbegin())); + EXPECT_TRUE(std::equal(vt.storage().cbegin(), vt.storage().cend(), rm.m_data.cbegin())); strides_type new_strides = {rm.m_strides[2], rm.m_strides[1], @@ -266,7 +270,7 @@ namespace xt vec.resize(rm.shape(), layout_type::row_major); assign_array(vec, rm.m_assigner); - EXPECT_TRUE(std::equal(vec.data().cbegin(), vec.data().cend(), rm.m_data.cbegin())); + EXPECT_TRUE(std::equal(vec.storage().cbegin(), vec.storage().cend(), rm.m_data.cbegin())); auto vec_copy = vec; @@ -275,7 +279,7 @@ namespace xt auto vt = transpose(vec, {1, 0, 2}); shape_type shape_new = {a[1], a[0], a[2]}; EXPECT_TRUE(std::equal(vt.shape().cbegin(), vt.shape().cend(), shape_new.begin())); - EXPECT_TRUE(std::equal(vt.data().cbegin(), vt.data().cend(), rm.m_data.cbegin())); + EXPECT_TRUE(std::equal(vt.storage().cbegin(), vt.storage().cend(), rm.m_data.cbegin())); strides_type new_strides = {rm.m_strides[1], rm.m_strides[0], @@ -342,7 +346,7 @@ namespace xt row_major_result rm; vec.resize(rm.m_shape, layout_type::row_major); assign_array(vec, rm.m_assigner); - EXPECT_TRUE(std::equal(vec.data().cbegin(), vec.data().cend(), rm.m_data.cbegin())); + EXPECT_TRUE(std::equal(vec.storage().cbegin(), vec.storage().cend(), rm.m_data.cbegin())); EXPECT_EQ(vec(0, 1, 1), vec(1, 1)); EXPECT_EQ(vec(2, 1, 3), vec(2, 2, 2, 1, 3)); test_bound_check(vec); @@ -353,7 +357,7 @@ namespace xt column_major_result cm; vec.resize(cm.m_shape, layout_type::column_major); assign_array(vec, cm.m_assigner); - EXPECT_TRUE(std::equal(vec.data().cbegin(), vec.data().cend(), cm.m_data.cbegin())); + EXPECT_TRUE(std::equal(vec.storage().cbegin(), vec.storage().cend(), cm.m_data.cbegin())); EXPECT_EQ(vec(0, 1, 1), vec(1, 1)); EXPECT_EQ(vec(2, 1, 3), vec(2, 2, 2, 1, 3)); test_bound_check(vec); @@ -364,7 +368,7 @@ namespace xt central_major_result cem; vec.resize(cem.m_shape, cem.m_strides); assign_array(vec, cem.m_assigner); - EXPECT_TRUE(std::equal(vec.data().cbegin(), vec.data().cend(), cem.m_data.cbegin())); + EXPECT_TRUE(std::equal(vec.storage().cbegin(), vec.storage().cend(), cem.m_data.cbegin())); EXPECT_EQ(vec(0, 1, 1), vec(1, 1)); EXPECT_EQ(vec(2, 1, 3), vec(2, 2, 2, 1, 3)); test_bound_check(vec); @@ -375,7 +379,7 @@ namespace xt unit_shape_result usr; vec.resize(usr.m_shape, layout_type::row_major); assign_array(vec, usr.m_assigner); - EXPECT_TRUE(std::equal(vec.data().cbegin(), vec.data().cend(), usr.m_data.cbegin())); + EXPECT_TRUE(std::equal(vec.storage().cbegin(), vec.storage().cend(), usr.m_data.cbegin())); EXPECT_EQ(vec(0, 1, 0), vec(1, 0)); EXPECT_EQ(vec(2, 0, 3), vec(2, 2, 2, 0, 3)); test_bound_check(vec); @@ -475,7 +479,7 @@ namespace xt row_major_result rm; vec.resize(rm.m_shape, layout_type::row_major); indexed_assign_array(vec, rm.m_assigner); - EXPECT_TRUE(std::equal(vec.data().cbegin(), vec.data().cend(), rm.m_data.cbegin())); + EXPECT_TRUE(std::equal(vec.storage().cbegin(), vec.storage().cend(), rm.m_data.cbegin())); EXPECT_EQ(vec(0, 1, 1), vec[index1]); EXPECT_EQ(vec(2, 1, 3), vec[index2]); } @@ -485,7 +489,7 @@ namespace xt column_major_result cm; vec.resize(cm.m_shape, layout_type::column_major); indexed_assign_array(vec, cm.m_assigner); - EXPECT_TRUE(std::equal(vec.data().cbegin(), vec.data().cend(), cm.m_data.cbegin())); + EXPECT_TRUE(std::equal(vec.storage().cbegin(), vec.storage().cend(), cm.m_data.cbegin())); EXPECT_EQ(vec(0, 1, 1), vec[index1]); EXPECT_EQ(vec(2, 1, 3), vec[index2]); } @@ -495,7 +499,7 @@ namespace xt central_major_result cem; vec.resize(cem.m_shape, cem.m_strides); indexed_assign_array(vec, cem.m_assigner); - EXPECT_TRUE(std::equal(vec.data().cbegin(), vec.data().cend(), cem.m_data.cbegin())); + EXPECT_TRUE(std::equal(vec.storage().cbegin(), vec.storage().cend(), cem.m_data.cbegin())); EXPECT_EQ(vec(0, 1, 1), vec[index1]); EXPECT_EQ(vec(2, 1, 3), vec[index2]); } @@ -505,7 +509,7 @@ namespace xt unit_shape_result usr; vec.resize(usr.m_shape, layout_type::row_major); indexed_assign_array(vec, usr.m_assigner); - EXPECT_TRUE(std::equal(vec.data().cbegin(), vec.data().cend(), usr.m_data.cbegin())); + EXPECT_TRUE(std::equal(vec.storage().cbegin(), vec.storage().cend(), usr.m_data.cbegin())); xindex id1 = {1, 0}; xindex id2 = {2, 2, 2, 0, 3}; EXPECT_EQ(vec(0, 1, 0), vec[id1]); @@ -580,7 +584,7 @@ namespace xt row_major_result rm; vecrm.resize(rm.m_shape, layout_type::row_major); std::copy(rm.data().cbegin(), rm.data().cend(), vecrm.template begin()); - EXPECT_TRUE(std::equal(rm.data().cbegin(), rm.data().cend(), vecrm.data().cbegin())); + EXPECT_TRUE(std::equal(rm.data().cbegin(), rm.data().cend(), vecrm.storage().cbegin())); //EXPECT_EQ(vecrm.template end(), vecrm.data().end()); } @@ -589,7 +593,7 @@ namespace xt column_major_result cm; veccm.resize(cm.m_shape, layout_type::column_major); std::copy(cm.data().cbegin(), cm.data().cend(), veccm.template begin()); - EXPECT_TRUE(std::equal(cm.data().cbegin(), cm.data().cend(), veccm.data().cbegin())); + EXPECT_TRUE(std::equal(cm.data().cbegin(), cm.data().cend(), veccm.storage().cbegin())); //EXPECT_EQ(veccm.template end(), veccm.data().end()); } } diff --git a/test/test_pyarray.cpp b/test/test_pyarray.cpp index d6c1d43..15ede7e 100644 --- a/test/test_pyarray.cpp +++ b/test/test_pyarray.cpp @@ -1,5 +1,6 @@ /*************************************************************************** -* Copyright (c) 2016, Johan Mabille and Sylvain Corlay * +* Copyright (c) Wolf Vollprecht, Johan Mabille and Sylvain Corlay * +* Copyright (c) QuantStack * * * * Distributed under the terms of the BSD 3-Clause License. * * * @@ -10,7 +11,8 @@ #include "xtensor-python/pyarray.hpp" -#include "xtensor/xarray.hpp" +#include "xtensor/containers/xarray.hpp" +#include "xtensor/views/xview.hpp" #include "test_common.hpp" @@ -18,19 +20,84 @@ namespace xt { using container_type = std::vector; + template + using ndarray = pyarray; + + void test1 (ndarrayconst& x) + { + ndarray y = x; + ndarray z = xt::zeros({10}); + } + + double compute(ndarray const& xs) + { + auto v = xt::view (xs, 0, xt::all()); + return v(0); + } + TEST(pyarray, initializer_constructor) { - pyarray t - {{{ 0, 1, 2}, - { 3, 4, 5}, - { 6, 7, 8}}, - {{ 9, 10, 11}, - {12, 13, 14}, - {15, 16, 17}}}; + pyarray r + {{{ 0, 1, 2}, + { 3, 4, 5}, + { 6, 7, 8}}, + {{ 9, 10, 11}, + {12, 13, 14}, + {15, 16, 17}}}; + + EXPECT_EQ(r.layout(), xt::layout_type::row_major); + EXPECT_EQ(r.dimension(), 3); + EXPECT_EQ(r(0, 0, 1), 1); + EXPECT_EQ(r.shape()[0], 2); + + pyarray c + {{{ 0, 1, 2}, + { 3, 4, 5}, + { 6, 7, 8}}, + {{ 9, 10, 11}, + {12, 13, 14}, + {15, 16, 17}}}; + + EXPECT_EQ(c.layout(), xt::layout_type::column_major); + EXPECT_EQ(c.dimension(), 3); + EXPECT_EQ(c(0, 0, 1), 1); + EXPECT_EQ(c.shape()[0], 2); + + pyarray d + {{{ 0, 1, 2}, + { 3, 4, 5}, + { 6, 7, 8}}, + {{ 9, 10, 11}, + {12, 13, 14}, + {15, 16, 17}}}; + + EXPECT_EQ(d.layout(), xt::layout_type::row_major); + EXPECT_EQ(d.dimension(), 3); + EXPECT_EQ(d(0, 0, 1), 1); + EXPECT_EQ(d.shape()[0], 2); + } + + TEST(pyarray, expression) + { + pyarray a = xt::empty({}); + + EXPECT_EQ(a.layout(), xt::layout_type::row_major); + EXPECT_EQ(a.dimension(), 0); + EXPECT_EQ(a.size(), 1); + + pyarray b = xt::empty({5}); - EXPECT_EQ(t.dimension(), 3); - EXPECT_EQ(t(0, 0, 1), 1); - EXPECT_EQ(t.shape()[0], 2); + EXPECT_EQ(b.layout(), xt::layout_type::row_major); + EXPECT_EQ(b.dimension(), 1); + EXPECT_EQ(b.size(), 5); + + pyarray c = xt::empty({5, 3}); + + EXPECT_EQ(c.layout(), xt::layout_type::row_major); + EXPECT_EQ(c.dimension(), 2); + EXPECT_EQ(c.size(), 15); + EXPECT_EQ(c.shape(0), 5); + EXPECT_EQ(c.shape(1), 3); } TEST(pyarray, shaped_constructor) @@ -42,7 +109,7 @@ namespace xt compare_shape(ra, rm); EXPECT_EQ(layout_type::row_major, ra.layout()); } - + { SCOPED_TRACE("column_major constructor"); column_major_result<> cm; @@ -52,6 +119,15 @@ namespace xt } } + TEST(pyarray, from_shape) + { + auto arr = pyarray::from_shape({5, 2, 6}); + auto exp_shape = std::vector{5, 2, 6}; + EXPECT_TRUE(std::equal(arr.shape().begin(), arr.shape().end(), exp_shape.begin())); + EXPECT_EQ(arr.shape().size(), 3); + EXPECT_EQ(arr.size(), 5 * 2 * 6); + } + TEST(pyarray, strided_constructor) { central_major_result<> cmr; @@ -68,7 +144,7 @@ namespace xt pyarray ra(rm.m_shape, value); compare_shape(ra, rm); std::vector vec(ra.size(), value); - EXPECT_TRUE(std::equal(vec.cbegin(), vec.cend(), ra.data().cbegin())); + EXPECT_TRUE(std::equal(vec.cbegin(), vec.cend(), ra.storage().cbegin())); } { @@ -78,7 +154,7 @@ namespace xt pyarray ca(cm.m_shape, value, layout_type::column_major); compare_shape(ca, cm); std::vector vec(ca.size(), value); - EXPECT_TRUE(std::equal(vec.cbegin(), vec.cend(), ca.data().cbegin())); + EXPECT_TRUE(std::equal(vec.cbegin(), vec.cend(), ca.storage().cbegin())); } } @@ -89,7 +165,7 @@ namespace xt pyarray cma(cmr.m_shape, cmr.m_strides, value); compare_shape(cma, cmr); std::vector vec(cma.size(), value); - EXPECT_TRUE(std::equal(vec.cbegin(), vec.cend(), cma.data().cbegin())); + EXPECT_TRUE(std::equal(vec.cbegin(), vec.cend(), cma.storage().cbegin())); } TEST(pyarray, copy_semantic) @@ -97,26 +173,26 @@ namespace xt central_major_result<> res; int value = 2; pyarray a(res.m_shape, res.m_strides, value); - + { SCOPED_TRACE("copy constructor"); pyarray b(a); compare_shape(a, b); - EXPECT_EQ(a.data(), b.data()); + EXPECT_EQ(a.storage(), b.storage()); a.data()[0] += 1; - EXPECT_NE(a.data()[0], b.data()[0]); + EXPECT_NE(a.storage()[0], b.storage()[0]); } { SCOPED_TRACE("assignment operator"); row_major_result<> r; pyarray c(r.m_shape, 0); - EXPECT_NE(a.data(), c.data()); + EXPECT_NE(a.storage(), c.storage()); c = a; compare_shape(a, c); - EXPECT_EQ(a.data(), c.data()); + EXPECT_EQ(a.storage(), c.storage()); a.data()[0] += 1; - EXPECT_NE(a.data()[0], c.data()[0]); + EXPECT_NE(a.storage()[0], c.storage()[0]); } } @@ -131,18 +207,18 @@ namespace xt pyarray tmp(a); pyarray b(std::move(tmp)); compare_shape(a, b); - EXPECT_EQ(a.data(), b.data()); + EXPECT_EQ(a.storage(), b.storage()); } { SCOPED_TRACE("move assignment"); row_major_result<> r; pyarray c(r.m_shape, 0); - EXPECT_NE(a.data(), c.data()); + EXPECT_NE(a.storage(), c.storage()); pyarray tmp(a); c = std::move(tmp); compare_shape(a, c); - EXPECT_EQ(a.data(), c.data()); + EXPECT_EQ(a.storage(), c.storage()); } } @@ -155,6 +231,18 @@ namespace xt EXPECT_EQ(c(0, 1), a1(0, 1) + a2(0, 1)); EXPECT_EQ(c(1, 0), a1(1, 0) + a2(1, 0)); EXPECT_EQ(c(1, 1), a1(1, 1) + a2(1, 1)); + + pyarray d = a1 + a2; + EXPECT_EQ(d(0, 0), a1(0, 0) + a2(0, 0)); + EXPECT_EQ(d(0, 1), a1(0, 1) + a2(0, 1)); + EXPECT_EQ(d(1, 0), a1(1, 0) + a2(1, 0)); + EXPECT_EQ(d(1, 1), a1(1, 1) + a2(1, 1)); + + pyarray e = a1 + a2; + EXPECT_EQ(e(0, 0), a1(0, 0) + a2(0, 0)); + EXPECT_EQ(e(0, 1), a1(0, 1) + a2(0, 1)); + EXPECT_EQ(e(1, 0), a1(1, 0) + a2(1, 0)); + EXPECT_EQ(e(1, 1), a1(1, 1) + a2(1, 1)); } TEST(pyarray, resize) @@ -197,6 +285,10 @@ namespace xt pyarray a; pyarray b; test_iterator(a, b); + + pyarray c; + bool truthy = std::is_same::value; + EXPECT_TRUE(truthy); } TEST(pyarray, initializer_list) @@ -208,7 +300,7 @@ namespace xt EXPECT_EQ(2, a1(1)); EXPECT_EQ(4, a2(1, 1)); } - + TEST(pyarray, zerod) { pyarray a; @@ -218,14 +310,28 @@ namespace xt TEST(pyarray, reshape) { pyarray a = {{1,2,3}, {4,5,6}}; - auto ptr = a.raw_data(); + auto ptr = a.data(); a.reshape({1, 6}); std::vector sc1({1, 6}); EXPECT_TRUE(std::equal(sc1.begin(), sc1.end(), a.shape().begin()) && a.shape().size() == 2); - EXPECT_EQ(ptr, a.raw_data()); + EXPECT_EQ(ptr, a.data()); a.reshape({6}); std::vector sc2 = {6}; EXPECT_TRUE(std::equal(sc2.begin(), sc2.end(), a.shape().begin()) && a.shape().size() == 1); - EXPECT_EQ(ptr, a.raw_data()); + EXPECT_EQ(ptr, a.data()); + } + + TEST(pyarray, view) + { + xt::pyarray arr = xt::zeros({ 10 }); + auto v = xt::view(arr, xt::all()); + EXPECT_EQ(v(0), 0.); + } + + TEST(pyarray, zerod_copy) + { + xt::pyarray arr = 2; + xt::pyarray arr2(arr); + EXPECT_EQ(arr(), arr2()); } } diff --git a/test/test_pyarray_traits.cpp b/test/test_pyarray_traits.cpp new file mode 100644 index 0000000..fe9ef6a --- /dev/null +++ b/test/test_pyarray_traits.cpp @@ -0,0 +1,166 @@ +/*************************************************************************** +* Copyright (c) Wolf Vollprecht, Johan Mabille and Sylvain Corlay * +* Copyright (c) QuantStack * +* * +* Distributed under the terms of the BSD 3-Clause License. * +* * +* The full license is in the file LICENSE, distributed with this software. * +****************************************************************************/ + +#include "gtest/gtest.h" + +#include "xtensor-python/pyarray.hpp" + + + +namespace xt +{ + namespace testing + { + class pyarray_traits: public ::testing::Test + { + protected: + + using dynamic_type = xt::pyarray; + using row_major_type = xt::pyarray; + using column_major_type = xt::pyarray; + + dynamic_type d1 = {{0., 1.}, {0., 10.}, {0., 100.}}; + dynamic_type d2 = {{0., 2.}, {0., 20.}, {0., 200.}}; + + row_major_type r1 = {{0., 1.}, {0., 10.}, {0., 100.}}; + row_major_type r2 = {{0., 2.}, {0., 20.}, {0., 200.}}; + + column_major_type c1 = {{0., 1.}, {0., 10.}, {0., 100.}}; + column_major_type c2 = {{0., 2.}, {0., 20.}, {0., 200.}}; + + template + bool test_has_strides(T const&) + { + return xt::has_strides::value; + } + + template + xt::layout_type test_result_layout(T const& a1, T const& a2) + { + auto tmp1 = pow(sin((a2 - a1) / 2.), 2.); + auto tmp2 = cos(a1); + return (tmp1 + tmp2).layout(); + } + + template + bool test_linear_assign(T const& a1, T const& a2) + { + auto tmp1 = pow(sin((a2 - a1) / 2.), 2.); + auto tmp2 = cos(a1); + T res = tmp1 + tmp2; + return xt::xassign_traits::linear_assign(res, tmp1 + tmp2, true); + } + + template + bool test_static_simd_linear_assign(T const& a1, T const& a2) + { + auto tmp1 = pow(sin((a2 - a1) / 2.), 2.); + auto tmp2 = cos(a1); + return xt::xassign_traits::simd_linear_assign(); + } + + template + bool test_dynamic_simd_linear_assign(T const& a1, T const& a2) + { + auto tmp1 = pow(sin((a2 - a1) / 2.), 2.); + auto tmp2 = cos(a1); + return xt::xassign_traits::simd_linear_assign(a1, tmp2); + } + + template + bool test_linear_static_layout(T const& a1, T const& a2) + { + auto tmp1 = pow(sin((a2 - a1) / 2.), 2.); + auto tmp2 = cos(a1); + return xt::detail::linear_static_layout(); + } + + template + bool test_contiguous_layout(T const& a1, T const& a2) + { + auto tmp1 = pow(sin((a2 - a1) / 2.), 2.); + auto tmp2 = cos(a1); + return decltype(tmp1)::contiguous_layout && decltype(tmp2)::contiguous_layout; + } + }; + + TEST_F(pyarray_traits, result_layout) + { + EXPECT_TRUE(d1.layout() == layout_type::row_major); + EXPECT_TRUE(test_result_layout(d1, d2) == layout_type::row_major); + + EXPECT_TRUE(r1.layout() == layout_type::row_major); + EXPECT_TRUE(test_result_layout(r1, r2) == layout_type::row_major); + + EXPECT_TRUE(c1.layout() == layout_type::column_major); + EXPECT_TRUE(test_result_layout(c1, c2) == layout_type::column_major); + } + + TEST_F(pyarray_traits, has_strides) + { + EXPECT_TRUE(test_has_strides(d1)); + EXPECT_TRUE(test_has_strides(r1)); + EXPECT_TRUE(test_has_strides(c1)); + } + + TEST_F(pyarray_traits, has_linear_assign) + { + EXPECT_TRUE(d2.has_linear_assign(d1.strides())); + EXPECT_TRUE(r2.has_linear_assign(r1.strides())); + EXPECT_TRUE(c2.has_linear_assign(c1.strides())); + } + + TEST_F(pyarray_traits, linear_assign) + { + EXPECT_TRUE(test_linear_assign(d1, d2)); + EXPECT_TRUE(test_linear_assign(r1, r2)); + EXPECT_TRUE(test_linear_assign(c1, c2)); + } + + TEST_F(pyarray_traits, static_simd_linear_assign) + { +#ifdef XTENSOR_USE_XSIMD + EXPECT_FALSE(test_static_simd_linear_assign(d1, d2)); + EXPECT_TRUE(test_static_simd_linear_assign(r1, r2)); + EXPECT_TRUE(test_static_simd_linear_assign(c1, c2)); +#else + EXPECT_FALSE(test_static_simd_linear_assign(d1, d2)); + EXPECT_FALSE(test_static_simd_linear_assign(r1, r2)); + EXPECT_FALSE(test_static_simd_linear_assign(c1, c2)); +#endif + } + + TEST_F(pyarray_traits, dynamic_simd_linear_assign) + { +#ifdef XTENSOR_USE_XSIMD + EXPECT_TRUE(test_dynamic_simd_linear_assign(d1, d2)); + EXPECT_TRUE(test_dynamic_simd_linear_assign(r1, r2)); + EXPECT_TRUE(test_dynamic_simd_linear_assign(c1, c2)); +#else + EXPECT_FALSE(test_dynamic_simd_linear_assign(d1, d2)); + EXPECT_FALSE(test_dynamic_simd_linear_assign(r1, r2)); + EXPECT_FALSE(test_dynamic_simd_linear_assign(c1, c2)); +#endif + } + + TEST_F(pyarray_traits, linear_static_layout) + { + EXPECT_FALSE(test_linear_static_layout(d1, d2)); + EXPECT_TRUE(test_linear_static_layout(r1, r2)); + EXPECT_TRUE(test_linear_static_layout(c1, c2)); + } + + TEST_F(pyarray_traits, contiguous_layout) + { + EXPECT_FALSE(test_contiguous_layout(d1, d2)); + EXPECT_TRUE(test_contiguous_layout(r1, r2)); + EXPECT_TRUE(test_contiguous_layout(c1, c2)); + } + } +} diff --git a/test/test_pytensor.cpp b/test/test_pytensor.cpp index 4748a04..c7b12cf 100644 --- a/test/test_pytensor.cpp +++ b/test/test_pytensor.cpp @@ -1,5 +1,6 @@ /*************************************************************************** -* Copyright (c) 2016, Johan Mabille and Sylvain Corlay * +* Copyright (c) Wolf Vollprecht, Johan Mabille and Sylvain Corlay * +* Copyright (c) QuantStack * * * * Distributed under the terms of the BSD 3-Clause License. * * * @@ -10,7 +11,8 @@ #include "xtensor-python/pytensor.hpp" -#include "xtensor/xtensor.hpp" +#include "xtensor/containers/xtensor.hpp" +#include "xtensor/views/xview.hpp" #include "test_common.hpp" @@ -20,13 +22,13 @@ namespace xt TEST(pytensor, initializer_constructor) { - pytensor t - {{{ 0, 1, 2}, - { 3, 4, 5}, - { 6, 7, 8}}, - {{ 9, 10, 11}, - {12, 13, 14}, - {15, 16, 17}}}; + pytensor t + {{{ 0, 1, 2}, + { 3, 4, 5}, + { 6, 7, 8}}, + {{ 9, 10, 11}, + {12, 13, 14}, + {15, 16, 17}}}; EXPECT_EQ(t.dimension(), 3); EXPECT_EQ(t(0, 0, 1), 1); EXPECT_EQ(t.shape()[0], 2); @@ -51,6 +53,27 @@ namespace xt } } + TEST(pytensor, from_shape) + { + auto arr = pytensor::from_shape({5, 2, 6}); + auto exp_shape = std::vector{5, 2, 6}; + EXPECT_TRUE(std::equal(arr.shape().begin(), arr.shape().end(), exp_shape.begin())); + EXPECT_EQ(arr.shape().size(), 3); + EXPECT_EQ(arr.size(), 5 * 2 * 6); + using pyt3 = pytensor; + std::vector shp = std::vector{5, 2}; + EXPECT_THROW(pyt3::from_shape(shp), std::runtime_error); + } + + TEST(pytensor, scalar_from_shape) + { + std::array shape; + auto a = pytensor::from_shape(shape); + pytensor b(1.2); + EXPECT_TRUE(a.size() == b.size()); + EXPECT_TRUE(xt::has_shape(a, b.shape())); + } + TEST(pytensor, strided_constructor) { central_major_result cmr; @@ -67,7 +90,7 @@ namespace xt pytensor ra(rm.m_shape, value); compare_shape(ra, rm); std::vector vec(ra.size(), value); - EXPECT_TRUE(std::equal(vec.cbegin(), vec.cend(), ra.data().cbegin())); + EXPECT_TRUE(std::equal(vec.cbegin(), vec.cend(), ra.storage().cbegin())); } { @@ -77,7 +100,7 @@ namespace xt pytensor ca(cm.m_shape, value, layout_type::column_major); compare_shape(ca, cm); std::vector vec(ca.size(), value); - EXPECT_TRUE(std::equal(vec.cbegin(), vec.cend(), ca.data().cbegin())); + EXPECT_TRUE(std::equal(vec.cbegin(), vec.cend(), ca.storage().cbegin())); } } @@ -88,7 +111,7 @@ namespace xt pytensor cma(cmr.m_shape, cmr.m_strides, value); compare_shape(cma, cmr); std::vector vec(cma.size(), value); - EXPECT_TRUE(std::equal(vec.cbegin(), vec.cend(), cma.data().cbegin())); + EXPECT_TRUE(std::equal(vec.cbegin(), vec.cend(), cma.storage().cbegin())); } TEST(pytensor, copy_semantic) @@ -101,9 +124,9 @@ namespace xt SCOPED_TRACE("copy constructor"); pytensor b(a); compare_shape(a, b); - EXPECT_EQ(a.data(), b.data()); + EXPECT_EQ(a.storage(), b.storage()); a.data()[0] += 1; - EXPECT_NE(a.data()[0], b.data()[0]); + EXPECT_NE(a.storage()[0], b.storage()[0]); } { @@ -113,9 +136,9 @@ namespace xt EXPECT_NE(a.data(), c.data()); c = a; compare_shape(a, c); - EXPECT_EQ(a.data(), c.data()); + EXPECT_EQ(a.storage(), c.storage()); a.data()[0] += 1; - EXPECT_NE(a.data()[0], c.data()[0]); + EXPECT_NE(a.storage()[0], c.storage()[0]); } } @@ -130,18 +153,18 @@ namespace xt pytensor tmp(a); pytensor b(std::move(tmp)); compare_shape(a, b); - EXPECT_EQ(a.data(), b.data()); + EXPECT_EQ(a.storage(), b.storage()); } { SCOPED_TRACE("move assignment"); row_major_result r; pytensor c(r.m_shape, 0); - EXPECT_NE(a.data(), c.data()); + EXPECT_NE(a.storage(), c.storage()); pytensor tmp(a); c = std::move(tmp); compare_shape(a, c); - EXPECT_EQ(a.data(), c.data()); + EXPECT_EQ(a.storage(), c.storage()); } } @@ -195,6 +218,10 @@ namespace xt pytensor a; pytensor b; test_iterator, pytensor, container_type>(a, b); + + pytensor c; + bool truthy = std::is_same::value; + EXPECT_TRUE(truthy); } TEST(pytensor, zerod) @@ -206,9 +233,42 @@ namespace xt TEST(pytensor, reshape) { pytensor a = {{1,2,3}, {4,5,6}}; - auto ptr = a.raw_data(); + auto ptr = a.data(); + a.reshape(a.shape()); // compilation check a.reshape({1, 6}); - EXPECT_EQ(ptr, a.raw_data()); - EXPECT_THROW(a.reshape({6}), std::runtime_error); + EXPECT_EQ(ptr, a.data()); + EXPECT_THROW(a.reshape(std::vector{6}), std::runtime_error); + // note this throws because std array has only 1 element initialized + // and the second element is `0`. + EXPECT_THROW(a.reshape({6, 5}), std::runtime_error); + } + + TEST(pytensor, view) + { + xt::pytensor arr = xt::zeros({ 10 }); + auto v = xt::view(arr, xt::all()); + EXPECT_EQ(v(0), 0.); + } + + TEST(pytensor, unary) + { + pytensor a = { 1, 2, 3 }; + pytensor res = -a; + pytensor ref = { -1, -2, -3 }; + EXPECT_EQ(ref(0), res(0)); + EXPECT_EQ(ref(1), res(1)); + EXPECT_EQ(ref(1), res(1)); + } + + TEST(pytensor, inplace_pybind11_overload) + { + // pybind11 overrrides a number of operators in pybind11::object. + // This is testing that the right overload is picked up. + pytensor a = { 1.0, 2.0, 3.0 }; + a /= 2; + pytensor ref = { 0.5, 1.0, 1.5 }; + EXPECT_EQ(ref(0), a(0)); + EXPECT_EQ(ref(1), a(1)); + EXPECT_EQ(ref(1), a(1)); } } diff --git a/test/test_pyvectorize.cpp b/test/test_pyvectorize.cpp index cc20c8e..30378e6 100644 --- a/test/test_pyvectorize.cpp +++ b/test/test_pyvectorize.cpp @@ -1,5 +1,6 @@ /*************************************************************************** -* Copyright (c) 2016, Johan Mabille and Sylvain Corlay * +* Copyright (c) Wolf Vollprecht, Johan Mabille and Sylvain Corlay * +* Copyright (c) QuantStack * * * * Distributed under the terms of the BSD 3-Clause License. * * * diff --git a/test/test_sfinae.cpp b/test/test_sfinae.cpp new file mode 100644 index 0000000..c9d4733 --- /dev/null +++ b/test/test_sfinae.cpp @@ -0,0 +1,53 @@ +/*************************************************************************** +* Copyright (c) Wolf Vollprecht, Johan Mabille and Sylvain Corlay * +* Copyright (c) QuantStack * +* * +* Distributed under the terms of the BSD 3-Clause License. * +* * +* The full license is in the file LICENSE, distributed with this software. * +****************************************************************************/ + +#include + +#include "gtest/gtest.h" +#include "xtensor-python/pytensor.hpp" +#include "xtensor-python/pyarray.hpp" +#include "xtensor/containers/xarray.hpp" +#include "xtensor/containers/xtensor.hpp" + +namespace xt +{ + template ::value, int> = 0> + inline bool sfinae_has_fixed_rank(E&&) + { + return false; + } + + template ::value, int> = 0> + inline bool sfinae_has_fixed_rank(E&&) + { + return true; + } + + TEST(sfinae, fixed_rank) + { + xt::pyarray a = {{9, 9, 9}, {9, 9, 9}}; + xt::pytensor b = {9, 9}; + xt::pytensor c = {{9, 9}, {9, 9}}; + + EXPECT_TRUE(sfinae_has_fixed_rank(a) == false); + EXPECT_TRUE(sfinae_has_fixed_rank(b) == true); + EXPECT_TRUE(sfinae_has_fixed_rank(c) == true); + } + + TEST(sfinae, get_rank) + { + xt::pytensor A = xt::zeros({2}); + xt::pytensor B = xt::zeros({2, 2}); + xt::pyarray C = xt::zeros({2, 2}); + + EXPECT_TRUE(xt::get_rank::value == 1ul); + EXPECT_TRUE(xt::get_rank::value == 2ul); + EXPECT_TRUE(xt::get_rank::value == SIZE_MAX); + } +} diff --git a/test_python/main.cpp b/test_python/main.cpp index 84c4cb2..d9b9e97 100644 --- a/test_python/main.cpp +++ b/test_python/main.cpp @@ -1,5 +1,6 @@ /*************************************************************************** -* Copyright (c) 2016, Johan Mabille and Sylvain Corlay * +* Copyright (c) Wolf Vollprecht, Johan Mabille and Sylvain Corlay * +* Copyright (c) QuantStack * * * * Distributed under the terms of the BSD 3-Clause License. * * * @@ -8,11 +9,15 @@ #include -#include "xtensor/xmath.hpp" -#include "xtensor/xarray.hpp" +#include "xtensor/core/xmath.hpp" +#include "xtensor/containers/xarray.hpp" +#include "xtensor/containers/xfixed.hpp" #define FORCE_IMPORT_ARRAY #include "xtensor-python/pyarray.hpp" +#include "xtensor-python/pytensor.hpp" #include "xtensor-python/pyvectorize.hpp" +#include "xtensor/containers/xadapt.hpp" +#include "xtensor/views/xstrided_view.hpp" namespace py = pybind11; using complex_t = std::complex; @@ -29,6 +34,49 @@ xt::pyarray example2(xt::pyarray& m) return m + 2; } +xt::xarray example3_xarray(const xt::xarray& m) +{ + return xt::transpose(m) + 2; +} + +xt::xarray example3_xarray_colmajor( + const xt::xarray& m) +{ + return xt::transpose(m) + 2; +} + +xt::xtensor example3_xtensor3(const xt::xtensor& m) +{ + return xt::transpose(m) + 2; +} + +xt::xtensor example3_xtensor2(const xt::xtensor& m) +{ + return xt::transpose(m) + 2; +} + +xt::xtensor example3_xtensor2_colmajor( + const xt::xtensor& m) +{ + return xt::transpose(m) + 2; +} + +xt::xtensor_fixed> example3_xfixed3(const xt::xtensor_fixed>& m) +{ + return xt::transpose(m) + 2; +} + +xt::xtensor_fixed> example3_xfixed2(const xt::xtensor_fixed>& m) +{ + return xt::transpose(m) + 2; +} + +xt::xtensor_fixed, xt::layout_type::column_major> example3_xfixed2_colmajor( + const xt::xtensor_fixed, xt::layout_type::column_major>& m) +{ + return xt::transpose(m) + 2; +} + // Readme Examples double readme_example1(xt::pyarray& m) @@ -60,6 +108,28 @@ auto no_complex_overload_reg(const double& a) { return a; } +// +// Operator examples +// +xt::pyarray array_addition(const xt::pyarray& m, const xt::pyarray& n) +{ + return m + n; +} + +xt::pyarray array_subtraction(xt::pyarray& m, xt::pyarray& n) +{ + return m - n; +} + +xt::pyarray array_multiplication(xt::pyarray& m, xt::pyarray& n) +{ + return m * n; +} + +xt::pyarray array_division(xt::pyarray& m, xt::pyarray& n) +{ + return m / n; +} // Vectorize Examples @@ -107,6 +177,136 @@ void dump_numpy_constant() std::cout << "NPY_UINT64 = " << NPY_UINT64 << std::endl; } +struct A +{ + double a; + int b; + char c; + std::array x; +}; + +struct B +{ + double a; + int b; +}; + +class C +{ +public: + using array_type = xt::xarray; + C() : m_array{0, 0, 0, 0} {} + array_type & array() { return m_array; } +private: + array_type m_array; +}; + +struct test_native_casters +{ + using array_type = xt::xarray; + array_type a = xt::ones({50, 50}); + + const auto & get_array() + { + return a; + } + + auto get_strided_view() + { + return xt::strided_view(a, {xt::range(0, 1), xt::range(0, 3, 2)}); + } + + auto get_array_adapter() + { + using shape_type = std::vector; + shape_type shape = {2, 2}; + shape_type stride = {3, 2}; + return xt::adapt(a.data(), 4, xt::no_ownership(), shape, stride); + } + + auto get_tensor_adapter() + { + using shape_type = std::array; + shape_type shape = {2, 2}; + shape_type stride = {3, 2}; + return xt::adapt(a.data(), 4, xt::no_ownership(), shape, stride); + } + + auto get_owning_array_adapter() + { + size_t size = 100; + int * data = new int[size]; + std::fill(data, data + size, 1); + + using shape_type = std::vector; + shape_type shape = {size}; + return xt::adapt(std::move(data), size, xt::acquire_ownership(), shape); + } +}; + +xt::pyarray dtype_to_python() +{ + A a1{123, 321, 'a', {1, 2, 3}}; + A a2{111, 222, 'x', {5, 5, 5}}; + + return xt::pyarray({a1, a2}); +} + +xt::pyarray dtype_from_python(xt::pyarray& b) +{ + if (b(0).a != 1 || b(0).b != 'p' || b(1).a != 123 || b(1).b != 'c') + { + throw std::runtime_error("FAIL"); + } + + b(0).a = 123.; + b(0).b = 'w'; + return b; +} + +void char_array(xt::pyarray& carr) +{ + if (strcmp(carr(2), "python")) + { + throw std::runtime_error("TEST FAILED!"); + } + std::fill(&carr(2)[0], &carr(2)[0] + 20, 0); + carr(2)[0] = 'c'; + carr(2)[1] = '+'; + carr(2)[2] = '+'; + carr(2)[3] = '\0'; +} + +void row_major_tensor(xt::pytensor& arg) +{ + if (!std::is_same::value) + { + throw std::runtime_error("TEST FAILED"); + } +} + +void col_major_array(xt::pyarray& arg) +{ + if (!std::is_same()), double*>::value) + { + throw std::runtime_error("TEST FAILED"); + } +} + +xt::pytensor xscalar(const xt::pytensor& arg) +{ + return xt::sum(arg); +} + +template +using ndarray = xt::pyarray; + +void test_rm(ndarrayconst& x) +{ + ndarray y = x; + ndarray z = xt::zeros({10}); +} + PYBIND11_MODULE(xtensor_python_test, m) { xt::import_numpy(); @@ -115,6 +315,14 @@ PYBIND11_MODULE(xtensor_python_test, m) m.def("example1", example1); m.def("example2", example2); + m.def("example3_xarray", example3_xarray); + m.def("example3_xarray_colmajor", example3_xarray_colmajor); + m.def("example3_xtensor3", example3_xtensor3); + m.def("example3_xtensor2", example3_xtensor2); + m.def("example3_xtensor2_colmajor", example3_xtensor2_colmajor); + m.def("example3_xfixed3", example3_xfixed3); + m.def("example3_xfixed2", example3_xfixed2); + m.def("example3_xfixed2_colmajor", example3_xfixed2_colmajor); m.def("complex_overload", no_complex_overload); m.def("complex_overload", complex_overload); @@ -124,6 +332,11 @@ PYBIND11_MODULE(xtensor_python_test, m) m.def("readme_example1", readme_example1); m.def("readme_example2", xt::pyvectorize(readme_example2)); + m.def("array_addition", array_addition); + m.def("array_subtraction", array_subtraction); + m.def("array_multiplication", array_multiplication); + m.def("array_division", array_division); + m.def("vectorize_example1", xt::pyvectorize(add)); m.def("rect_to_polar", xt::pyvectorize([](complex_t x) { return std::abs(x); })); @@ -132,6 +345,8 @@ PYBIND11_MODULE(xtensor_python_test, m) return a.shape() == b.shape(); }); + m.def("test_rm", test_rm); + m.def("int_overload", int_overload); m.def("int_overload", int_overload); m.def("int_overload", int_overload); @@ -142,4 +357,46 @@ PYBIND11_MODULE(xtensor_python_test, m) m.def("int_overload", int_overload); m.def("dump_numpy_constant", dump_numpy_constant); + + // Register additional dtypes + PYBIND11_NUMPY_DTYPE(A, a, b, c, x); + PYBIND11_NUMPY_DTYPE(B, a, b); + + m.def("dtype_to_python", dtype_to_python); + m.def("dtype_from_python", dtype_from_python); + m.def("char_array", char_array); + + m.def("col_major_array", col_major_array); + m.def("row_major_tensor", row_major_tensor); + + m.def("xscalar", xscalar); + + py::class_(m, "C") + .def(py::init<>()) + .def_property_readonly( + "copy", + [](C & self) { return self.array(); } + ) + .def_property_readonly( + "ref", + [](C & self) -> C::array_type & { return self.array(); } + ) + ; + + m.def("simple_array", [](xt::pyarray) { return 1; } ); + m.def("simple_tensor", [](xt::pytensor) { return 2; } ); + + m.def("diff_shape_overload", [](xt::pytensor a) { return 1; }); + m.def("diff_shape_overload", [](xt::pytensor a) { return 2; }); + + py::class_(m, "test_native_casters") + .def(py::init<>()) + .def("get_array", &test_native_casters::get_array, py::return_value_policy::reference_internal) // memory managed by the class instance + .def("get_strided_view", &test_native_casters::get_strided_view, py::keep_alive<0, 1>()) // keep_alive<0, 1>() => do not free "self" before the returned view + .def("get_array_adapter", &test_native_casters::get_array_adapter, py::keep_alive<0, 1>()) // keep_alive<0, 1>() => do not free "self" before the returned adapter + .def("get_tensor_adapter", &test_native_casters::get_tensor_adapter, py::keep_alive<0, 1>()) // keep_alive<0, 1>() => do not free "self" before the returned adapter + .def("get_owning_array_adapter", &test_native_casters::get_owning_array_adapter) // auto memory management as the adapter owns its memory + .def("view_keep_alive_member_function", [](test_native_casters & self, xt::pyarray & a) // keep_alive<0, 2>() => do not free second parameter before the returned view + {return xt::reshape_view(a, {a.size(), });}, + py::keep_alive<0, 2>()); } diff --git a/test_python/setup.py b/test_python/setup.py index aea0f49..895f60d 100644 --- a/test_python/setup.py +++ b/test_python/setup.py @@ -1,5 +1,6 @@ ############################################################################ -# Copyright (c) 2016, Johan Mabille and Sylvain Corlay # +# Copyright (c) Wolf Vollprecht, Johan Mabille and Sylvain Corlay # +# Copyright (c) QuantStack # # # # Distributed under the terms of the BSD 3-Clause License. # # # @@ -46,6 +47,7 @@ def __str__(self): ['main.cpp'], include_dirs=[ # Path to pybind11 headers + '../include/', get_pybind_include(), get_pybind_include(user=True), # Path to numpy headers @@ -73,13 +75,13 @@ def has_flag(compiler, flagname): def cpp_flag(compiler): - """Return the -std=c++14 compiler flag and errors when the flag is + """Return the -std=c++17 compiler flag and errors when the flag is no available. """ - if has_flag(compiler, '-std=c++14'): - return '-std=c++14' + if has_flag(compiler, '-std=c++20'): + return '-std=c++20' else: - raise RuntimeError('C++14 support is required by xtensor!') + raise RuntimeError('C++17 support is required by xtensor!') class BuildExt(build_ext): @@ -90,7 +92,7 @@ class BuildExt(build_ext): } if sys.platform == 'darwin': - c_opts['unix'] += ['-stdlib=libc++', '-mmacosx-version-min=10.7'] + c_opts['unix'] += ['-stdlib=libc++', '-mmacosx-version-min=10.13'] def build_extensions(self): ct = self.compiler.compiler_type @@ -102,6 +104,7 @@ def build_extensions(self): opts.append('-fvisibility=hidden') elif ct == 'msvc': opts.append('/DVERSION_INFO=\\"%s\\"' % self.distribution.get_version()) + opts.append('/std:c++20') for ext in self.extensions: ext.extra_compile_args = opts build_ext.build_extensions(self) diff --git a/test_python/test_pyarray.py b/test_python/test_pyarray.py index 7aa67a8..4e7d5eb 100644 --- a/test_python/test_pyarray.py +++ b/test_python/test_pyarray.py @@ -1,5 +1,6 @@ ############################################################################ -# Copyright (c) 2016, Johan Mabille and Sylvain Corlay # +# Copyright (c) Wolf Vollprecht, Johan Mabille and Sylvain Corlay # +# Copyright (c) QuantStack # # # # Distributed under the terms of the BSD 3-Clause License. # # # @@ -21,7 +22,9 @@ import xtensor_python_test as xt import numpy as np -class ExampleTest(TestCase): +class XtensorTest(TestCase): + def test_rm(self): + xt.test_rm(np.array([10], dtype=int)) def test_example1(self): self.assertEqual(4, xt.example1([4, 5, 6])) @@ -32,6 +35,75 @@ def test_example2(self): y = xt.example2(x) np.testing.assert_allclose(y, res, 1e-12) + def test_example3(self): + x = np.arange(2 * 3).reshape(2, 3) + xc = np.asfortranarray(x) + y = np.arange(2 * 3 * 4).reshape(2, 3, 4) + v = y[1:, 1:, 0] + z = np.arange(2 * 3 * 4 * 5).reshape(2, 3, 4, 5) + np.testing.assert_array_equal(xt.example3_xarray(x), x.T + 2) + np.testing.assert_array_equal(xt.example3_xarray_colmajor(xc), xc.T + 2) + np.testing.assert_array_equal(xt.example3_xtensor3(y), y.T + 2) + np.testing.assert_array_equal(xt.example3_xtensor2(x), x.T + 2) + np.testing.assert_array_equal(xt.example3_xtensor2(y[1:, 1:, 0]), v.T + 2) + np.testing.assert_array_equal(xt.example3_xtensor2_colmajor(xc), xc.T + 2) + + np.testing.assert_array_equal(xt.example3_xfixed3(y), y.T + 2) + np.testing.assert_array_equal(xt.example3_xfixed2(x), x.T + 2) + np.testing.assert_array_equal(xt.example3_xfixed2_colmajor(xc), xc.T + 2) + + with self.assertRaises(TypeError): + xt.example3_xtensor3(x) + + with self.assertRaises(TypeError): + xt.example3_xfixed3(x) + + with self.assertRaises(TypeError): + x = np.arange(3*2).reshape(3, 2) + xt.example3_xfixed2(x) + def test_broadcast_addition(self): + x = np.array([[2., 3., 4., 5.]]) + y = np.array([[1., 2., 3., 4.], + [1., 2., 3., 4.], + [1., 2., 3., 4.]]) + res = np.array([[3., 5., 7., 9.], + [3., 5., 7., 9.], + [3., 5., 7., 9.]]) + z = xt.array_addition(x, y) + np.testing.assert_allclose(z, res, 1e-12) + def test_broadcast_subtraction(self): + x = np.array([[4., 5., 6., 7.]]) + y = np.array([[4., 3., 2., 1.], + [4., 3., 2., 1.], + [4., 3., 2., 1.]]) + res = np.array([[0., 2., 4., 6.], + [0., 2., 4., 6.], + [0., 2., 4., 6.]]) + z = xt.array_subtraction(x, y) + np.testing.assert_allclose(z, res, 1e-12) + + def test_broadcast_multiplication(self): + x = np.array([[1., 2., 3., 4.]]) + y = np.array([[3., 2., 3., 2.], + [3., 2., 3., 2.], + [3., 2., 3., 2.]]) + res = np.array([[3., 4., 9., 8.], + [3., 4., 9., 8.], + [3., 4., 9., 8.]]) + z = xt.array_multiplication(x, y) + np.testing.assert_allclose(z, res, 1e-12) + + def test_broadcast_division(self): + x = np.array([[8., 6., 4., 2.]]) + y = np.array([[2., 2., 2., 2.], + [2., 2., 2., 2.], + [2., 2., 2., 2.]]) + res = np.array([[4., 3., 2., 1.], + [4., 3., 2., 1.], + [4., 3., 2., 1.]]) + z = xt.array_division(x, y) + np.testing.assert_allclose(z, res, 1e-12) + def test_vectorize(self): x1 = np.array([[0, 1], [2, 3]]) x2 = np.array([0, 1]) @@ -65,7 +137,7 @@ def test_readme_example2(self): x = np.arange(15).reshape(3, 5) y = [1, 2, 3, 4, 5] z = xt.readme_example2(x, y) - np.testing.assert_allclose(z, + np.testing.assert_allclose(z, [[-0.540302, 1.257618, 1.89929 , 0.794764, -1.040465], [-1.499227, 0.136731, 1.646979, 1.643002, 0.128456], [-1.084323, -0.583843, 0.45342 , 1.073811, 0.706945]], 1e-5) @@ -87,3 +159,163 @@ def test_int_overload(self): b = xt.int_overload(np.ones((10), dtype)) self.assertEqual(str(dtype.__name__), b) + def test_dtype(self): + var = xt.dtype_to_python() + self.assertEqual(var.dtype.names, ('a', 'b', 'c', 'x')) + + exp_dtype = { + 'a': (np.dtype('float64'), 0), + 'b': (np.dtype('int32'), 8), + 'c': (np.dtype('int8'), 12), + 'x': (np.dtype(('