diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index e773a61..0000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,290 +0,0 @@ -name: Build and publish - -on: - pull_request: - push: - branches: - - main - tags: - - 'v*' - -jobs: - upload: - name: Upload to PyPI - if: github.repository == 'micro-manager/pymmcore' - runs-on: ubuntu-latest - needs: [sdist, wheels-manylinux, wheels-winmac] - steps: - - name: Download wheels and sdist - uses: actions/download-artifact@v3 - - - name: Collect wheels and sdist - run: | - mkdir dist - mv pymmcore-sdist/* dist/ - mv pymmcore-wheels-manylinux/* dist/ - mv pymmcore-wheels-win-mac/* dist/ - ls dist - - - name: Upload release to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') - with: - user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} - - - sdist: - name: sdist - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Checkout submodules - run: | - git submodule sync --recursive - git submodule update --init --force --recursive --depth=1 - - - name: Install tools and dependencies - run: | - sudo apt-get install -y python3-dev python3-pip python3-venv - python3 -m venv venv - source venv/bin/activate - pip install --upgrade pip build - - - name: Create source distribution - run: | - source venv/bin/activate - python -m build --sdist --outdir dist/ . - - - uses: actions/upload-artifact@v3 - with: - name: pymmcore-sdist - path: dist - - - wheels-manylinux: - strategy: - fail-fast: true - matrix: - manylinux-version: ['2014'] - arch: [i686, x86_64] - include: - - arch: i686 - numpy-versions: cp38-cp38 1.17.3 cp39-cp39 1.19.3 - - arch: x86_64 - numpy-versions: cp38-cp38 1.17.3 cp39-cp39 1.19.3 cp310-cp310 1.21.3 cp311-cp311 1.23.2 - - name: manylinux${{ matrix.manylinux-version }}_${{ matrix.arch }} - - runs-on: ubuntu-latest - - env: - AUDITWHEEL_PLAT: manylinux${{ matrix.manylinux-version }}_${{ matrix.arch }} - DOCKER_IMAGE: quay.io/pypa/manylinux${{ matrix.manylinux-version }}_${{ matrix.arch }} - NUMPY_VERSIONS: ${{ matrix.numpy-versions }} - - steps: - - uses: actions/checkout@v3 - - - name: Checkout submodules - run: | - git submodule sync --recursive - git submodule update --init --force --recursive --depth=1 - - - name: Pull image - run: | - docker pull $DOCKER_IMAGE - - - name: Build - run: | - docker run -e NUMPY_VERSIONS -e AUDITWHEEL_PLAT -v $(pwd):/io $DOCKER_IMAGE /io/manylinux/build.sh - - - uses: actions/upload-artifact@v3 - with: - name: pymmcore-wheels-manylinux - path: wheelhouse - - - wheels-winmac: - strategy: - fail-fast: true - matrix: - os: [Windows, macOS] - python-version: ['3.8', '3.9', '3.10', '3.11'] - python-arch: [x64, x86] - macos-deployment-target: ['10.9'] - msvc-version: ['14.1'] # VS2017 - include: - - os: Windows - runner: windows-latest - - os: macOS - runner: macOS-latest - - python-version: '3.8' - mac-python-version: 3.8.10 - mac-python-installer-macos-version: macosx10.9 - numpy-version: 1.17.3 - - python-version: '3.9' - mac-python-version: 3.9.7 - mac-python-installer-macos-version: macosx10.9 - numpy-version: 1.19.3 - - python-version: '3.10' - mac-python-version: 3.10.0 - mac-python-installer-macos-version: macos11 - numpy-version: 1.21.3 - - python-version: '3.11' - mac-python-version: 3.11.0 - mac-python-installer-macos-version: macos11 - numpy-version: 1.23.2 - - python-arch: x64 - msvc-arch: amd64 - - python-arch: x86 - msvc-arch: x86 - exclude: - - os: macOS - python-arch: x86 - - os: Windows # NumPy has 64-bit only for Windows wheels >= cp310 - python-version: '3.10' - python-arch: x86 - - name: ${{ matrix.os }} Python ${{ matrix.python-version }} ${{ matrix.python-arch }} - - runs-on: ${{ matrix.runner }} - - env: - MACOSX_DEPLOYMENT_TARGET: ${{ matrix.macos-deployment-target }} - - steps: - - uses: actions/checkout@v3 - - - name: Checkout submodules - run: | - git submodule sync --recursive - git submodule update --init --force --recursive --depth=1 - - - name: Find Visual Studio (Windows) - if: matrix.os == 'Windows' - shell: pwsh - run: | - $VsDir = (& "${Env:ProgramFiles(x86)}/Microsoft Visual Studio/Installer/vswhere" -latest -property installationPath) - echo "VCVARSALL_DIR=$VsDir/VC/Auxiliary/Build" >>$Env:GITHUB_ENV - - - name: Install Python (generic) - uses: actions/setup-python@v4 - if: matrix.os != 'macOS' - with: - python-version: ${{ matrix.python-version }} - architecture: ${{ matrix.python-arch }} - - - name: Install Python (macOS) - if: matrix.os == 'macOS' - run: | - curl -L -o python.pkg https://www.python.org/ftp/python/${{ matrix.mac-python-version }}/python-${{ matrix.mac-python-version }}-${{ matrix.mac-python-installer-macos-version }}.pkg - sudo installer -pkg python.pkg -target / - /Library/Frameworks/Python.framework/Versions/${{ matrix.python-version }}/bin/python3 -m venv venv - - - name: Install tools (Windows) - if: matrix.os == 'Windows' - run: | - choco install -y swig - - - name: Install tools (macOS) - if: matrix.os == 'macOS' - run: | - brew install swig - - - name: Install dependencies (Windows) - if: matrix.os == 'Windows' - run: | - python -m pip install --upgrade pip - python -m pip install --upgrade setuptools wheel - python -m pip install numpy==${{ matrix.numpy-version }} - - - name: Install dependencies (Unix) - if: matrix.os != 'Windows' - run: | - source venv/bin/activate - python -m pip install --upgrade pip - python -m pip install --upgrade setuptools wheel - python -m pip install numpy==${{ matrix.numpy-version }} - - - name: Package and extract sources (Windows) - if: matrix.os == 'Windows' - run: | - python setup.py sdist --format=zip - mkdir tmp - Expand-Archive -Path dist/pymmcore-*.zip -DestinationPath tmp - mv tmp/pymmcore-* tmp/pymmcore - - - name: Package and extract sources (Unix) - if: matrix.os == 'macOS' - run: | - source venv/bin/activate - python setup.py sdist - mkdir tmp - tar xvzf dist/pymmcore-*.tar.gz -C tmp - mv tmp/pymmcore-* tmp/pymmcore - - - name: Build wheel (Windows) - if: matrix.os == 'Windows' - shell: cmd - env: - MSSdk: 1 - DISTUTILS_USE_SDK: 1 - PY_VCRUNTIME_REDIST: 1 - run: | - pushd "%VCVARSALL_DIR%" - call vcvarsall.bat ${{ matrix.msvc-arch }} -vcvars_ver=${{ matrix.msvc-version }} - popd - cd tmp\pymmcore - python setup.py build_ext - python setup.py build - python setup.py bdist_wheel - - - name: Build wheel (macOS) - if: matrix.os == 'macOS' - env: - CC: clang - CXX: clang++ - CFLAGS: -fvisibility=hidden -Wno-unused-variable - run: | - source venv/bin/activate - cd tmp/pymmcore - python setup.py build_ext -j2 - python setup.py build - python setup.py bdist_wheel - - - name: Log undefined symbols (macOS) - if: matrix.os == 'macOS' - run: | - cd tmp/pymmcore - PYMOD=$(echo build/lib.*/pymmcore/_pymmcore_swig.*.so) - - echo "$PYMOD:" - echo 'Weak symbols:' - nm -mu $PYMOD |c++filt |grep ' weak ' # This is never empty - echo '-- end of weak symbols --' - - echo 'Undefined symbols not containing Py:' - nm -mu $PYMOD |c++filt |grep 'dynamically looked up' |grep -v _Py && exit 1 - echo '-- end of non-Py dynamically looked up symbols --' - - - name: Smoke test (Windows) - if: matrix.os == 'Windows' - run: | - cd .. # Avoid picking up pymmcore.py from cwd - python -m pip install (Get-Item pymmcore/tmp/pymmcore/dist/pymmcore-*.whl).FullName - python pymmcore/smoketest/smoke.py - - - name: Smoke test (Unix) - if: matrix.os != 'Windows' - run: | - source venv/bin/activate - cd .. # Avoid picking up pymmcore.py from cwd - python -m pip install pymmcore/tmp/pymmcore/dist/pymmcore-*.whl - python pymmcore/smoketest/smoke.py - - - uses: actions/upload-artifact@v3 - with: - name: pymmcore-wheels-win-mac - path: tmp/pymmcore/dist diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..a7390d7 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,100 @@ +name: Build & deploy + +on: + workflow_dispatch: + pull_request: + push: + branches: + - main + tags: + - "v*" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + # check that sdist contains all files and that extra files + # are explicitly ignored in manifest or pyproject + check-manifest: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: "recursive" + - name: Check manifest + run: pipx run check-manifest + + build_wheels: + name: Build wheels on ${{ matrix.os }} ${{ matrix.macos_arch }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-22.04, windows-2022] + include: + - os: macos-11 + macos_arch: "x86_64" + - os: macos-11 + macos_arch: "arm64" + + steps: + - uses: actions/checkout@v4 + with: + submodules: "recursive" + + - name: Build wheels + uses: pypa/cibuildwheel@v2.16 + env: + CIBW_ARCHS_MACOS: "${{ matrix.macos_arch }}" + + - uses: actions/upload-artifact@v3 + with: + path: ./wheelhouse/*.whl + + build_sdist: + name: Build source distribution + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: "recursive" + + - name: Build sdist + run: | + pip install -U pip build check-manifest + check-manifest + python -m build --sdist + + - uses: actions/upload-artifact@v3 + with: + path: dist/*.tar.gz + + upload_pypi: + needs: [build_wheels, build_sdist] + runs-on: ubuntu-latest + # upload to PyPI on every tag + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + + # https://docs.pypi.org/trusted-publishers/ + permissions: + id-token: write # for trusted publishing on PyPi + contents: write # allows writing releases + + steps: + - uses: actions/download-artifact@v3 + with: + # unpacks default artifact into dist/ + # if `name: artifact` is omitted, the action will create extra parent dir + name: artifact + path: dist + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + - uses: softprops/action-gh-release@v1 + with: + generate_release_notes: true + files: "./dist/*" diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml deleted file mode 100644 index 26484d4..0000000 --- a/.github/workflows/ubuntu.yml +++ /dev/null @@ -1,73 +0,0 @@ -name: Check build on Ubuntu - -# This workflow checks that our setup.py script works on Ubuntu, using -# system-provided dependencies. The binaries are not expected to work on any -# other Linux distribution. - -on: - pull_request: - push: - branches: - - main - tags: - - 'v*' - -jobs: - build: - strategy: - matrix: - runner: [ubuntu-20.04, ubuntu-22.04] - - name: ${{ matrix.runner }} build check - - runs-on: ${{ matrix.runner }} - - steps: - - - uses: actions/checkout@v3 - - - name: Checkout submodules - run: | - git submodule sync --recursive - git submodule update --init --force --recursive --depth=1 - - - name: Install tools and dependencies - run: | - sudo apt-get update - sudo apt-get install -y python3-dev python3-pip python3-venv - sudo apt-get install -y build-essential swig - python3 -m venv venv - source venv/bin/activate - pip install --upgrade pip - pip install --upgrade setuptools numpy - - - name: Package and extract sources - run: | - source venv/bin/activate - python setup.py sdist - mkdir tmp - tar xvzf dist/pymmcore-*.tar.gz -C tmp - mv tmp/pymmcore-* tmp/pymmcore - - - name: Build wheel - env: - CFLAGS: -Wno-deprecated - run: | - source venv/bin/activate - cd tmp/pymmcore - python setup.py build_ext -j2 - python setup.py build - python setup.py install - - - name: Check for undefined symbols - run: | - cd tmp/pymmcore - echo 'Missing symbols:' - ldd -r build/lib.*/pymmcore/_pymmcore_swig.*.so |grep '^undefined symbol:' |grep -v Py && exit 1 - echo '-- end of missing symbols --' - - - name: Smoke test - run: | - source venv/bin/activate - cd tmp/pymmcore - python ../../smoketest/smoke.py diff --git a/MANIFEST.in b/MANIFEST.in index af12579..d4a77dc 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,12 @@ -include pymmcore/py.typed recursive-include * *.pyi recursive-include mmCoreAndDevices/MMDevice *.h *.cpp recursive-include mmCoreAndDevices/MMCore *.h *.cpp + prune mmCoreAndDevices/MMDevice/unittest prune mmCoreAndDevices/MMCore/unittest +prune mmCoreAndDevices/MMCoreJ_wrap +prune mmCoreAndDevices/DeviceAdapters +prune mmCoreAndDevices/m4 +prune mmCoreAndDevices/.github +recursive-exclude mmCoreAndDevices *.txt *.md *.am .project \ + *.vcxproj* *.sln *.props *.cdt* secret-device-* diff --git a/README.md b/README.md index 94c788f..a520712 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,20 @@ -pymmcore: Python bindings for MMCore -==================================== +# pymmcore: Python bindings for MMCore The pymmcore package provides Python 3.x bindings to Micro-Manager's MMCore (the low-level device control/acquisition interface). Using pymmcore, you can control and acquire images from all of the microscope -devices supported by Micro-Manager, but without the GUI application. +devices supported by Micro-Manager, but without the GUI application or a Java +runtime. Not to be confused with [pycro-manager](https://github.com/micro-manager/pycro-manager), which allows -control of the entire Micro-Manager application, including its Java APIs, and -more. +control of the entire Java Micro-Manager application, including its Java APIs, +and more. You might also be interested in -[pymmcore-plus](https://pymmcore-plus.readthedocs.io) which wraps this library and provides extra functionality. +[pymmcore-plus](https://pymmcore-plus.readthedocs.io) which wraps this library +and provides extra functionality including an acquisition engine. Note: pymmcore is similar to the legacy MMCorePy module (Python 2.x only), previously distributed with the Micro-Manager application. However, the Python @@ -26,43 +27,36 @@ Because pymmcore is distributed separately from Micro-Manager, it needs to be "pointed" at an existing Micro-Manager installation to access device adapters. (See the example below.) +## Installing -Installing ----------- - -Windows (Python 3.8-3.11), macOS (Python 3.8-3.11, 64-bit), and Linux (Python -3.8-3.11) are supported. Only 64-bit is supported for Python 3.10 and later. +Suports Python 3.8 or later and Windows, macOS, and Linux (all 64-bit). ``` -python -m pip install --user pymmcore +pip install pymmcore ``` -You can leave out the `--user` if installing into a virtual environment -(recommended). Or install via conda: + ``` conda install -c conda-forge pymmcore ``` -Installation by `pip` should use binary wheels. If `pip` falls back to building -from source code, it will probably fail. If this happens in a supported -environment, please [file a -bug](https://github.com/micro-manager/pymmcore/issues). To manually build from -source, the scripts in `.github/workflows` should serve as a starting point. - -You also need a working installation of the Micro-Manager application. +You also need a working installation of the Micro-Manager device adapters. +(for a convenient way to install that programmatically, see +the [`mmcore install` command in pymmcore plus](https://pymmcore-plus.github.io/pymmcore-plus/install/#installing-micro-manager-device-adapters)) - -Quick example -------------- +## Quick example ```python import pymmcore import os.path +import os mm_dir = "C:/Program Files/Micro-Manager-2.0.x" mmc = pymmcore.CMMCore() + +os.environ["PATH"] += os.pathsep.join(["", mm_dir]) # adviseable on Windows mmc.setDeviceAdapterSearchPaths([mm_dir]) mmc.loadSystemConfiguration(os.path.join(mm_dir, "MMConfig_demo.cfg")) @@ -70,14 +64,12 @@ mmc.snapImage() mmc.getImage() ``` -We do not currently have Python-specific documentation for MMCore. The [Java -documentation](https://micro-manager.org/apidoc/mmcorej/latest/) is probably -the best resource (start at the class `CMMCore`). There is also [C++ +We do not currently have Python-specific documentation for MMCore, but +the [pymmcore-plus documentation](https://pymmcore-plus.github.io/pymmcore-plus/api/cmmcoreplus) +includes the [pymmcore.CMMCore class](https://pymmcore-plus.github.io/pymmcore-plus/api/cmmcoreplus/#pymmcore.CMMCore). There is also [C++ documentation](https://micro-manager.org/apidoc/MMCore/latest/). - -Matching Micro-Manager and pymmcore versions --------------------------------------------- +## Matching Micro-Manager and pymmcore versions The version number of pymmcore is independent of the Micro-Manager version number; instead it tracks the MMCore and device interface versions. @@ -91,6 +83,7 @@ viewed in **Help** > **About Micro-Manager**. The device interface version of a given pymmcore version is the fourth part in the version number, and can also be viewed as follows: + ```python import pymmcore pymmcore.CMMCore().getAPIVersionInfo() @@ -111,9 +104,7 @@ parts of the pymmcore version.) For a list of device interface versions for each pymmcore version, see the [Releases](https://github.com/micro-manager/pymmcore/releases) page. - -Loading device adapters on Windows ----------------------------------- +## Loading device adapters on Windows The majority of device adapters should load once `setDeviceAdapterSearchPaths()` has been called with the correct directories, @@ -137,15 +128,11 @@ additional DLLs at a later time. Please report any cases where the Micro-Manager application can load a configuration but pymmcore cannot, even when using the above methods. - -Code of Conduct ---------------- +## Code of Conduct This project is covered by the [Micro-Manager Code of Conduct](https://github.com/micro-manager/micro-manager/blob/master/CodeOfConduct.md). - -License -------- +## License The license for pymmcore itself is LGPL 2.1 (see `LICENSE.txt`). The MMCore component of Micro-Manager (which gets built into pymmcore) is also under the diff --git a/maintainer-notes.md b/maintainer-notes.md index db8ad62..1b18813 100644 --- a/maintainer-notes.md +++ b/maintainer-notes.md @@ -1,5 +1,4 @@ -Versioning scheme ------------------ +## Versioning scheme Cf. PEP 440. @@ -9,27 +8,74 @@ and works with device adapters built for device interface version 69. The final suffix can be incremented to create a new release for improvements to the pymmcore wrapper. +``` +pymmcore v10.1.1.69.0 + | | | + | | +----> pymmcore-specific suffix + | +-------> MMDevice device interface version + +--------------> MMCore version (major.minor.patch) +``` + (Note that the device interface version can change without a change to the MMCore version, although this is relatively uncommon. Also, we are leaving out the module interface version, which rarely changes.) -The correspondence to MMCore and device interface versions is checked in the -smoke test. +The correspondence to MMCore and device interface versions is checked in +`tests/test_mmcore.py`. Note that we can support multiple MMCore versions, possibly retroactively, by maintaining separate branches; this can ease transition when the device interface version changes. Such branches should be named `mmcore-x.y.z.w`. When upgrading the MMCore version (by bumping the mmCoreAndDevices submodule -commit), the pymmcore version in `setup.cfg` should be updated together. +commit), the pymmcore version in `_version.py` should be updated in synchrony. +The versioning for the python package is taken dynamically from that file +in the `[tool.setuptools.dynamic]` table in `pyproject.toml`. + +## Building Binary Wheels and Source Distributions + +The package can be built in a few ways: + +1. Use [cibuildwheel](https://cibuildwheel.readthedocs.io/en/stable/). + This is the method used by the GitHub Actions CI workflow (configuration + is in `pyproject.toml`). You can [run it locally](https://cibuildwheel.readthedocs.io/en/stable/setup/#local) as well + if you have Docker installed: + + ```sh + pip install cibuildwheel + # example + cibuildwheel --platform macos + ``` + Or, to build a specific platform/python version: + ```sh + cibuildwheel --only cp310-macosx_x86_64 + ``` + + The wheels will be placed in the `wheelhouse` directory. +2. Use the [build](https://pypi.org/project/build/) package -Release procedure ------------------ + ```sh + pip install build + python -m build + ``` + + This will build wheel an sdist and wheel for the current platform and + Python version, and place them in the `dist` directory. + +3. Use `pip install -e .` + This will build the extension module in-place and allow you to run tests, + but will not build a wheel or sdist. Note that if you do this, you will + need to rerun it each time you change the extension module. + + + +## Release procedure Prepare two commits, one removing `.dev0` from the version and a subsequent one bumping the patch version and re-adding `.dev0`. Tag the former with `v` prefixed to the version: + ```bash git checkout main @@ -40,21 +86,37 @@ git tag -a v1.2.3.42.4 -m Release vim pymmcore/_version.py # Set version to 1.2.3.42.5.dev0 git commit -a -m 'Version back to dev' -git push origin v1.2.3.42.4 +git push upstream --follow-tags git push ``` -This triggers a build, since our GitHub workflow builds on push, including when -it's an annotated tag. The build, when successful, automatically uploads to -PyPI when the tag name starts with `v`. +This triggers a build in [the ci.yml workflow](.github/workflows/ci.yml) and +the presence of a tag starting with "v" triggers a deployment to PyPI (using +[trusted publisher](https://docs.pypi.org/trusted-publishers/) authentication.) + +Pushing the tag also creates a GitHub release with auto-generated release notes +and the binary wheels attached. -Pushing the tag also creates a GitHub release, which can be edited to add -binaries. Upload the Windows, macOS, and manylinux wheels and source -distribution as a backup and second source. +## Dependency and tool versions +- The minimum version of python supported is declared in `pypyproject.toml`, + in the `[project.requires-python]` section. +- The build-time versions of numpy are in `pyproject.toml`, in the + `[build-system.requires]` section. +- The run-time numpy dependency is declared in `pyproject.toml`, in the + `[project.dependencies]` section. +- Wheels are built with `cibuildwheel`, and the various wheel versions are + determined by the settings in the `[tool.cibuildwheel]` section of + `pyproject.toml`. +- _We_ should provide wheels for all Python versions we claim to support, + built agains the oldest NumPy version that we claim to support. Thus, any + issue with the build or our CI will limit the lowest supported versions. -ABI Compatibility ------------------ +- Swig. + - Swig 4.x should be used. + - Swig 1.x generates code that is no longer compatible with Python 3.x. + +## ABI Compatibility - The Python platform and ABI compatibility is all handled by the Wheel system. (But see below re MSVC versions.) @@ -65,11 +127,35 @@ ABI Compatibility In practice, we should use the oldest NumPy for which wheels are available on PyPI for the given Python version (and all 3 platforms): + - Python 3.8 - NumPy 1.17.3 - Python 3.9 - NumPy 1.19.3 - - Python 3.10 - NumPy 1.21.3 (Windows: amd64 only) + - Python 3.10 - NumPy 1.21.3 - Python 3.11 - NumPy 1.23.2 + - Python 3.12 - Numpy 1.26.0 + + Those versions are reflected in the `[build-system.requires]` section of + `pyproject.toml`, which takes care of creating the appropriate build + environment for the wheel. + +## Building with debug symbols on Windows + +Since there is no easy way to pass compile and linker options to `build_clib`, +the easiest hack is to edit the local Python installation's +`Lib/distutils/_msvccompiler.py` to add the compiler flag `/Zi` and linker flag +`/DEBUG:FULL` (see the method `initialize`). This produces `vc140.pdb`. + +(The "normal" method would be to run `setup.py build_clib` and `setup.py +build_ext` with the `--debug` option, and run with `python_d.exe`. But then we +would need a debug build of NumPy, which is hard to build on Windows.) + + +### Legacy Build Notes +Many of these notes are probably obviated by the use of cibuildwheel... but +are kept for reference. + +
### Windows @@ -82,6 +168,7 @@ ABI Compatibility Python prints the MSVC version used to build itself when started. This version may change with the patch version of Python. Here are a few examples: + - Python 3.8.1 (64-bit): MSC v.1916 = VS2017 - Python 3.9.1 (64-bit): MSC v.1927 = VS2019 - Python 3.8.7 (64-bit): MSC v.1928 = VS2019 @@ -105,7 +192,6 @@ ABI Compatibility Windows installers are designed for non-admin installation, we technically should. - ### macOS - `MACOSX_DEPLOYMENT_TARGET` should be set to match the Python.org Python we @@ -122,45 +208,12 @@ ABI Compatibility are "dynamically looked up", other than those starting with `_Py` or `__Py`. There should be none if the build is correct. - ### Linux - The manylinux docker images appear to solve all our problems. -Dependency and tool versions ----------------------------- - -- The Python and NumPy version requirements in `setup.py` should be set so that - `pip` just works. - - NumPy wheels for the Python-NumPy version combination should be available - on PyPI (for mac/linux/windows) for the versions we support. - - _We_ should provide wheels for all Python versions we claim to support, - built agains the oldest NumPy version that we claim to support. Thus, any - issue with the build or our CI will limit the lowest supported versions. - - The required version ranges can be made platform-specific if necessary (see - setuptools docs) - -- Swig. - - Swig 1.x generates code that is no longer compatible with Python 3.x. - - Swig 4.x should be used. - - -Building with debug symbols on Windows --------------------------------------- - -Since there is no easy way to pass compile and linker options to `build_clib`, -the easiest hack is to edit the local Python installation's -`Lib/distutils/_msvccompiler.py` to add the compiler flag `/Zi` and linker flag -`/DEBUG:FULL` (see the method `initialize`). This produces `vc140.pdb`. - -(The "normal" method would be to run `setup.py build_clib` and `setup.py -build_ext` with the `--debug` option, and run with `python_d.exe`. But then we -would need a debug build of NumPy, which is hard to build on Windows.) - - -Resources ---------- +### Resources - [Windows Compilers](https://wiki.python.org/moin/WindowsCompilers) on Python Wiki - [MacPython: Spinning wheels](https://github.com/MacPython/wiki/wiki/Spinning-wheels) (macOS ABI) @@ -173,3 +226,6 @@ Resources - Unmaintained Apple [tech note](https://developer.apple.com/library/archive/technotes/tn2064/_index.html) describing `MACOSX_DEPLOYMENT_TARGET` + + +
\ No newline at end of file diff --git a/manylinux/build.sh b/manylinux/build.sh deleted file mode 100755 index 176d050..0000000 --- a/manylinux/build.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/bin/bash - -set -e -x - -test -n "$PARALLEL" || PARALLEL=-j2 - -cd / - - -git clone https://github.com/swig/swig.git -pushd swig -git checkout rel-4.0.1 -curl -fLO https://iweb.dl.sourceforge.net/project/pcre/pcre/8.45/pcre-8.45.tar.gz || \ - curl -fLJO https://sourceforge.net/projects/pcre/files/pcre/8.45/pcre-8.45.tar.gz/download -./Tools/pcre-build.sh -./autogen.sh -./configure -make $PARALLEL -make install -popd - - -# NUMPY_VERSIONS contains alternating ABI tags and NumPy versions. -# Convert it to an associative array. -numpy_versions=($NUMPY_VERSIONS) -declare -A numpy_version_map -abitags=() # To preserve ordering -for ((i=0; i<${#numpy_versions[@]}; i+=2)); do - abitag=${numpy_versions[i]} - numpy_version=${numpy_versions[i+1]} - - abitags+=($abitag) - numpy_version_map[$abitag]=$numpy_version -done - - -cd /io -for abitag in ${abitags[@]}; do - numpy_version=${numpy_version_map[$abitag]} - pybin=/opt/python/$abitag/bin - - # Avoid altering NumPy compile+link (-fvisibility=hidden will break it) - export CFLAGS= - export LDFLAGS= - $pybin/pip install --upgrade pip - $pybin/pip install --upgrade setuptools wheel numpy==${numpy_version} - - # Package and extract sources to ensure sdist is correct - rm -rf tmp dist/pymmcore-*.tar.gz - $pybin/python setup.py sdist - mkdir tmp - tar xzf dist/pymmcore-*.tar.gz -C tmp - cd tmp/pymmcore-* - - export CFLAGS="-fvisibility=hidden -Wno-deprecated -Wno-unused-variable" - export LDFLAGS="-Wl,--strip-debug" # Sane file size - $pybin/python setup.py build_ext $PARALLEL - $pybin/python setup.py build - $pybin/python setup.py bdist_wheel - mkdir -p /io/prelim-wheels - mv dist/*.whl /io/prelim-wheels -done - - -# Update ABI tag -cd /io -compgen -G "prelim-wheels/*.whl" # Fail if none built -mkdir -p wheelhouse -for wheel in prelim-wheels/*.whl; do - auditwheel show $wheel - auditwheel repair $wheel -w wheelhouse -done - - -# Smoke test -cd / -for abitag in ${abitags[@]}; do - pybin=/opt/python/$abitag/bin - $pybin/pip install pymmcore --no-index -f /io/wheelhouse - $pybin/python /io/smoketest/smoke.py -done diff --git a/pyproject.toml b/pyproject.toml index c6e2d39..5c914d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,69 @@ +# https://peps.python.org/pep-0517/ [build-system] -requires = ["setuptools", "numpy>=1.12.0"] +requires = [ + "setuptools >=61.0.0", + "numpy==1.14.5; python_version=='3.7'", + "numpy==1.17.3; python_version=='3.8'", + "numpy==1.19.3; python_version=='3.9'", + "numpy==1.21.3; python_version=='3.10'", + "numpy==1.23.2; python_version=='3.11'", + "numpy==1.26.0; python_version=='3.12'", +] build-backend = "setuptools.build_meta" + + +# https://peps.python.org/pep-0621/ +[project] +name = "pymmcore" +description = "Python bindings for MMCore, Micro-Manager's device control layer" +dynamic = ["version"] +readme = "README.md" +requires-python = ">=3.7" +license = { text = "BSD 3-Clause License" } +authors = [{ name = "Micro-Manager Team" }] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: GNU Lesser General Public License v2 (LGPLv2)", + "Operating System :: MacOS :: MacOS X", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: Implementation :: CPython", + "Topic :: Scientific/Engineering", + "Topic :: System :: Hardware :: Hardware Drivers", + "Typing :: Typed", +] +dependencies = ["numpy >=1.12.0"] + +[project.urls] +homepage = "https://micro-manager.org" +repository = "https://github.com/micro-manager/pymmcore" + + +[tool.setuptools.dynamic] +version = { attr = "pymmcore._version.__version__" } + +[tool.setuptools.packages.find] +include = ["pymmcore*"] + +[tool.setuptools.package-data] +"*" = ["py.typed", ".pyi"] + + +[tool.cibuildwheel] +# Skip 32-bit builds, musllinux, and PyPy wheels on all platforms +# Note: use of PTHREAD_MUTEX_RECURSIVE_NP in DeviceThreads.h +# is specific to glibc and not available in musl-libc +skip = ["*-manylinux_i686", "*-musllinux*", "*-win32", "pp*"] +build = ["cp38-*", "cp39-*", "cp310-*", "cp311-*", "cp312-*"] +test-requires = "pytest" +test-command = 'pytest "{project}/tests" -v' +test-skip = "*-macosx_arm64" + +[tool.cibuildwheel.macos] +# https://cibuildwheel.readthedocs.io/en/stable/faq/#apple-silicon +archs = ["x86_64", "arm64"] + +[tool.check-manifest] +ignore = [".editorconfig", "Dockerfile", "maintainer-notes.md", ".gitmodules"] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index be368d2..0000000 --- a/setup.cfg +++ /dev/null @@ -1,19 +0,0 @@ -[metadata] -name = pymmcore -version = attr: pymmcore._version.__version__ -author = Micro-Manager Team -url = https://github.com/micro-manager/pymmcore -description = Python bindings for MMCore, Micro-Manager's device control layer -long_description = file: README.md -long_description_content_type = text/markdown -classifiers = - Development Status :: 5 - Production/Stable - Intended Audience :: Science/Research - License :: OSI Approved :: GNU Lesser General Public License v2 (LGPLv2) - Operating System :: MacOS :: MacOS X - Operating System :: Microsoft :: Windows - Operating System :: POSIX :: Linux - Programming Language :: Python :: 3 - Programming Language :: Python :: Implementation :: CPython - Topic :: Scientific/Engineering - Topic :: System :: Hardware :: Hardware Drivers diff --git a/setup.py b/setup.py index d7a9388..ca49fc4 100644 --- a/setup.py +++ b/setup.py @@ -18,20 +18,23 @@ # # Author: Mark A. Tsuchida -import distutils.file_util -import distutils.util -import glob -import numpy import os import os.path -import setuptools +import platform +from pathlib import Path + +import numpy import setuptools.command.build_ext import setuptools.command.build_py +from setuptools import Extension, setup -pkg_name = 'pymmcore' -swig_mod_name = 'pymmcore_swig' -ext_mod_name = '.'.join((pkg_name, '_' + swig_mod_name)) - +PKG_NAME = "pymmcore" +SWIG_MOD_NAME = "pymmcore_swig" +IS_WINDOWS = platform.system() == "Windows" +IS_MACOS = platform.system() == "Darwin" +ROOT = Path(__file__).parent +MMCorePath = ROOT / "mmCoreAndDevices" / "MMCore" +MMDevicePath = ROOT / "mmCoreAndDevices" / "MMDevice" # We build MMCore from sources, into the Python extension. MMCore depends on # MMDevice. However, we need to build MMDevice separately from MMCore, because @@ -44,147 +47,83 @@ # .py file gets missed. class build_py(setuptools.command.build_py.build_py): def run(self): - self.run_command('build_ext') + self.run_command("build_ext") super().run() # Customize 'build_ext' to trigger 'build_clib' first. class build_ext(setuptools.command.build_ext.build_ext): def run(self): - self.run_command('build_clib') + self.run_command("build_clib") super().run() -is_windows = distutils.util.get_platform().startswith('win') -is_macos = distutils.util.get_platform().startswith('macosx') - -windows_defines = [ - ('_CRT_SECURE_NO_WARNINGS', None), - - # These would not be necessary if _WIN32 or _MSC_VER were used correctly. - ('WIN32', None), - ('_WINDOWS', None), - - # See DeviceUtils.h - ('MMDEVICE_NO_GETTIMEOFDAY', None), -] - - mmdevice_build_info = { - 'sources': glob.glob('mmCoreAndDevices/MMDevice/*.cpp'), - 'include_dirs': [ - 'mmCoreAndDevices/MMDevice', - ], - 'macros': [ - ('MODULE_EXPORTS', None), - ], + "sources": [str(x.relative_to(ROOT)) for x in MMDevicePath.glob("*.cpp")], + "include_dirs": ["mmCoreAndDevices/MMDevice"], + "macros": [("MODULE_EXPORTS", None)], } -if is_windows: - mmdevice_build_info['macros'].extend(windows_defines) - - -mmcore_source_globs = [ - 'mmCoreAndDevices/MMCore/*.cpp', - 'mmCoreAndDevices/MMCore/Devices/*.cpp', - 'mmCoreAndDevices/MMCore/LibraryInfo/*.cpp', - 'mmCoreAndDevices/MMCore/LoadableModules/*.cpp', - 'mmCoreAndDevices/MMCore/Logging/*.cpp', -] - -mmcore_sources = [] -for g in mmcore_source_globs: - mmcore_sources += glob.glob(g) -if is_windows: - mmcore_sources = [f for f in mmcore_sources if 'Unix' not in f] +if IS_WINDOWS: + define_macros = [ + ("_CRT_SECURE_NO_WARNINGS", None), + # These would not be necessary if _WIN32 or _MSC_VER were used correctly. + ("WIN32", None), + ("_WINDOWS", None), + # See DeviceUtils.h + ("MMDEVICE_NO_GETTIMEOFDAY", None), + ] + mmdevice_build_info["macros"].extend(define_macros) else: - mmcore_sources = [f for f in mmcore_sources if 'Windows' not in f] - + define_macros = [] -mmcore_libraries = [ - 'MMDevice', +omit = ["unittest"] + (["Unix"] if IS_WINDOWS else ["Windows"]) +mmcore_sources = [ + str(x.relative_to(ROOT)) + for x in MMCorePath.rglob("*.cpp") + if all(o not in str(x) for o in omit) ] -if is_windows: - mmcore_libraries.extend([ - 'Iphlpapi', - 'Advapi32', - ]) -else: - mmcore_libraries.extend([ - 'dl', - ]) +mmcore_libraries = ["MMDevice"] +if IS_WINDOWS: + mmcore_libraries.extend(["Iphlpapi", "Advapi32"]) +else: + mmcore_libraries.extend(["dl"]) -if not is_windows: - cflags = [ - '-std=c++14', - ] - if 'CFLAGS' in os.environ: - cflags.insert(0, os.environ['CFLAGS']) - os.environ['CFLAGS'] = ' '.join(cflags) +if not IS_WINDOWS: + cflags = ["-std=c++14"] + if "CFLAGS" in os.environ: + cflags.insert(0, os.environ["CFLAGS"]) + os.environ["CFLAGS"] = " ".join(cflags) # MMCore on macOS currently requires these frameworks (for a feature that # should be deprecated). Frameworks need to appear on the linker command line # before the object files, so extra_link_args doesn't work. -if is_macos: - ldflags = [ - '-framework', 'CoreFoundation', - '-framework', 'IOKit', - ] - if 'LDFLAGS' in os.environ: - ldflags.insert(0, os.environ['LDFLAGS']) - os.environ['LDFLAGS'] = ' '.join(ldflags) - - -mmcore_defines = [] -if is_windows: - mmcore_defines.extend(windows_defines) +if IS_MACOS: + ldflags = ["-framework", "CoreFoundation", "-framework", "IOKit"] + if "LDFLAGS" in os.environ: + ldflags.insert(0, os.environ["LDFLAGS"]) + os.environ["LDFLAGS"] = " ".join(ldflags) -mmcore_extension = setuptools.Extension( - ext_mod_name, - sources=mmcore_sources + [ - os.path.join(pkg_name, swig_mod_name + '.i'), - ], +mmcore_extension = Extension( + f"{PKG_NAME}._{SWIG_MOD_NAME}", + sources=mmcore_sources + [os.path.join(PKG_NAME, f"{SWIG_MOD_NAME}.i")], swig_opts=[ - '-c++', - '-py3', - '-builtin', - '-I./mmCoreAndDevices/MMDevice', - '-I./mmCoreAndDevices/MMCore', - ], - include_dirs=[ - numpy.get_include(), + "-c++", + "-py3", + "-builtin", + "-I./mmCoreAndDevices/MMDevice", + "-I./mmCoreAndDevices/MMCore", ], + include_dirs=[numpy.get_include()], libraries=mmcore_libraries, - define_macros=mmcore_defines, + define_macros=define_macros, ) - -# See maintainer notes! -python_req = '>=3.6' -numpy_req = '>=1.12.0' - - -setuptools.setup( - packages=setuptools.find_packages(include=(pkg_name + '*',)), +setup( ext_modules=[mmcore_extension], - libraries=[ - ('MMDevice', mmdevice_build_info), - ], - python_requires=python_req, - setup_requires=[ - 'numpy' + numpy_req, - ], - install_requires=[ - 'numpy' + numpy_req, - ], - cmdclass={ - 'build_ext': build_ext, - 'build_py': build_py, - }, - package_data={ - 'pymmcore': ['*.pyi', 'py.typed'], - }, + libraries=[("MMDevice", mmdevice_build_info)], + cmdclass={"build_ext": build_ext, "build_py": build_py}, ) diff --git a/smoketest/smoke.py b/smoketest/smoke.py deleted file mode 100644 index d18ca9a..0000000 --- a/smoketest/smoke.py +++ /dev/null @@ -1,31 +0,0 @@ -import pymmcore - - -# -# At the moment, the only test is that our version numbering is correct -# - -pymmcore_version = pymmcore.__version__.split('.') -print("Version: {}".format(pymmcore_version)) - -mmc = pymmcore.CMMCore() - -# getVersionInfo() returns a string like "MMCore version 10.1.1" -mmcore_version = mmc.getVersionInfo().split()[-1].split('.') -print("MMCore version: {}".format(mmcore_version)) - -for i in range(3): - if pymmcore_version[i] != mmcore_version[i]: - raise AssertionError('Version mismatch between pymmcore and MMCore') - - -# getAPIVersionInfo() returns a string like -# "Device API version 69, Module API version 10" -dev_if_version = mmc.getAPIVersionInfo().split(',')[0].split()[-1] -mod_if_version = mmc.getAPIVersionInfo().split()[-1] -print("MMDevice device interface version: {}".format(dev_if_version)) -print("MMDevice module interface version: {}".format(mod_if_version)) - -if pymmcore_version[3] != dev_if_version: - raise AssertionError( - 'Version mismatch between pymmcore and device interface') diff --git a/tests/test_mmcore.py b/tests/test_mmcore.py new file mode 100644 index 0000000..dfc0f6c --- /dev/null +++ b/tests/test_mmcore.py @@ -0,0 +1,17 @@ +import pymmcore + + +def test_core(): + # __version__ will be something like '10.4.0.71.1.dev0' + pymmcore_version = pymmcore.__version__.split(".") + + mmc = pymmcore.CMMCore() + + # something like 'MMCore version 10.4.0' + version_info = mmc.getVersionInfo() + assert pymmcore_version[:3] == version_info.split()[-1].split(".") + + # something like 'Device API version 71, Module API version 10' + api_version_info = mmc.getAPIVersionInfo() + dev_interface_version = api_version_info.split(",")[0].split()[-1] + assert pymmcore_version[3] == dev_interface_version