Migrate to poetry (#827)

IAMconsortium · Mar 7, 2024 · e77080c · e77080c
1 parent ddbb88e
commit e77080c
Show file tree

Hide file tree

Showing 16 changed files with 4,973 additions and 206 deletions.
diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml
@@ -16,15 +16,39 @@ jobs:
     - uses: actions/checkout@v4
 
     - name: Set up Python
+      id: setup-python
       uses: actions/setup-python@v5
       with:
         python-version: '3.10'
 
     - name: Install Pandoc
-      uses: r-lib/actions/setup-pandoc@v1
+      uses: r-lib/actions/setup-pandoc@v2
 
-    - name: Install dependencies and package for building the docs
-      run: pip install -e .[tutorials,optional_plotting,docs]
+    #------------------------------
+    #  install & configure poetry  
+    #------------------------------
+    - name: Install Poetry
+      uses: snok/install-poetry@v1
+      with:
+        virtualenvs-create: true
+        virtualenvs-in-project: true
+        installer-parallel: true
+
+    #------------------------------------
+    #  load cached venv if cache exists
+    #------------------------------------
+    - name: Load cached venv
+      id: cached-poetry-dependencies
+      uses: actions/cache@v4
+      with:
+        path: .venv
+        key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
+
+    #------------------------
+    #  install root project
+    #------------------------
+    - name: Install library
+      run: poetry install --no-interaction --extras "docs optional_plotting tutorials"
 
     - name: Build the docs
-      run: make --directory=docs html
+      run: poetry run make --directory=docs html
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -35,19 +35,49 @@ jobs:
         python-version: ${{ matrix.python-version }}
 
     - name: Install Pandoc
-      uses: r-lib/actions/setup-pandoc@v1
+      uses: r-lib/actions/setup-pandoc@v2
 
-    - name: Install dependencies and package
-      run: |
-        python -m pip install --upgrade pip
-        pip install -e .[tests,optional_plotting,optional_io_formats,tutorials,docs]
+    #------------------------------
+    #  install & configure poetry  
+    #------------------------------
+    - name: Install Poetry
+      uses: snok/install-poetry@v1
+      with:
+        virtualenvs-create: true
+        virtualenvs-in-project: true
+        installer-parallel: true
+
+    #------------------------------------------
+    #  update dependencies to latest versions
+    #------------------------------------------
+    - name: Update dependencies
+      run: poetry update --lock
+
+    #------------------------------------
+    #  load cached venv if cache exists
+    #------------------------------------
+    - name: Load cached venv
+      id: cached-poetry-dependencies
+      uses: actions/cache@v4
+      with:
+        path: .venv
+        key: venv-${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}
+
+    #------------------------------------------------
+    #  install dependencies if cache does not exist
+    #------------------------------------------------
+    - name: Install dependencies
+      if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+      run: poetry install --no-interaction --extras "docs optional_io_formats optional_plotting tests tutorials" --no-root
+
+    #------------------------
+    #  install root project
+    #------------------------
+    - name: Install library
+      run: poetry install --no-interaction --extras "optional_io_formats optional_plotting tests tutorials" --only-root
 
     - name: Test with pytest (including Matplotlib)
-      run: |
-        pytest tests --mpl
+      run: poetry run pytest tests --mpl
 
     - name: Build the docs
-      run: |
-        cd docs
-        make html
-        cd ..
+      run: poetry run make --directory=docs html
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -19,20 +19,21 @@ jobs:
 
     - uses: actions/setup-python@v5
 
-    - name: Cache Python packages
-      uses: actions/cache@v4
+    #------------------------------
+    #  install & configure poetry  
+    #------------------------------
+    - name: Install Poetry
+      uses: snok/install-poetry@v1
       with:
-        path: |
-          ~/.cache/pip
-        key: publish-${{ runner.os }}
-
-    - name: Upgrade pip, wheel, setuptools-scm
-      run: python -m pip install --upgrade pip wheel setuptools-scm twine
+        virtualenvs-create: true
+        virtualenvs-in-project: true
+        installer-parallel: true
 
+    - name: Install poetry dynamic versioning plugin
+      run: poetry self add "poetry-dynamic-versioning[plugin]"
+
     - name: Build package
-      run: |
-        python3 setup.py bdist_wheel sdist
-        twine check dist/*
+      run: poetry build
 
     - name: Publish to TestPyPI
       uses: pypa/[email protected]

diff --git a/.github/workflows/pytest-legacy.yml b/.github/workflows/pytest-legacy.yml
@@ -18,17 +18,55 @@ jobs:
     - uses: actions/checkout@v4
 
     - name: Set up Python
+      id: setup-python
       uses: actions/setup-python@v5
       with:
         python-version: '3.10'
 
-    - name: Install specific out-dated version of dependencies
+    #------------------------------
+    #  install & configure poetry
+    #------------------------------
+    - name: Install Poetry
+      uses: snok/install-poetry@v1
+      with:
+        virtualenvs-create: true
+        virtualenvs-in-project: true
+        installer-parallel: true
+
+    - name: Enforce usage of specific out-dated versions of dependencies
       # Update the package requirements when changing minimum dependency versions
       # Please also add a section "Dependency changes" to the release notes
-      run: pip install pandas==2.0.0 numpy==1.23.0 matplotlib==3.6.0 iam-units==2020.4.21 xlrd==2.0.1 pint==0.13
+      # Don't install packages, just update lock file to see if a cache exists
+      run: |
+        poetry add [email protected] --lock
+        poetry add [email protected] --lock
+        poetry add [email protected] --lock
+        poetry add [email protected] --lock
+        poetry add [email protected] --lock
+        poetry add [email protected] --optional --lock
+
+    #------------------------------------
+    #  load cached venv if cache exists
+    #------------------------------------
+    - name: Load cached venv
+      id: cached-poetry-dependencies
+      uses: actions/cache@v4
+      with:
+        path: .venv
+        key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
+
+    #------------------------------------------------
+    #  install dependencies if cache does not exist
+    #------------------------------------------------
+    - name: Install dependencies
+      if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+      run: poetry install --no-interaction --extras "optional_io_formats optional_plotting tests tutorials" --no-root
 
-    - name: Install other dependencies and package
-      run: pip install .[tests,optional_plotting,optional_io_formats,tutorials]
+    #------------------------
+    #  install root project
+    #------------------------
+    - name: Install library
+      run: poetry install --no-interaction --extras "optional_io_formats optional_plotting tests tutorials" --only-root
 
     - name: Test with pytest
-      run: pytest tests
+      run: poetry run pytest tests
diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml
@@ -23,6 +23,10 @@ jobs:
         - '3.12'
 
       fail-fast: false
+
+    defaults:
+      run:
+        shell: bash
 
     runs-on: ${{ matrix.os }}
     name: ${{ matrix.os }} py${{ matrix.python-version }}
@@ -31,26 +35,49 @@ jobs:
     - uses: actions/checkout@v4
 
     - name: Set up Python ${{ matrix.python-version }}
+      id: setup-python
       uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
-    - name: Install dependencies and package
-      run: pip install .[tests,optional_plotting,optional_io_formats,tutorials]
+    #------------------------------
+    #  install & configure poetry
+    #------------------------------
+    - name: Install Poetry
+      uses: snok/install-poetry@v1
+      with:
+        virtualenvs-create: true
+        virtualenvs-in-project: true
+        installer-parallel: true
+
+    #------------------------------------
+    #  load cached venv if cache exists
+    #------------------------------------
+    - name: Load cached venv
+      id: cached-poetry-dependencies
+      uses: actions/cache@v4
+      with:
+        path: .venv
+        key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
+    #------------------------
+    #  install your project
+    #------------------------
+    - name: Install library
+      run: poetry install --no-interaction --extras "optional_io_formats optional_plotting tests tutorials"
 
     # run tests without Matplotlib & CodeCode tests on earlier Python versions
     - name: Test with pytest
       if: ${{ matrix.python-version != '3.12' }}
-      run: pytest tests
+      run: poetry run pytest tests
 
     # run tests with Matplotlib & CodeCov on latest Python version
     - name: Test with pytest including Matplotlib & Codecov
       if: ${{ matrix.python-version == '3.12' }}
-      run: pytest tests --mpl --cov=./ --cov-report=xml
+      run: poetry run pytest tests --mpl --cov=./ --cov-report=xml
 
     - name: Upload coverage report to Codecov
       if: ${{ matrix.os == 'ubuntu-latest' && matrix.python-version == '3.12' }}
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v4
       with:
         file: ./coverage.xml
         env_vars: ${{ matrix.os }} py${{ matrix.python-version }}
diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
@@ -33,6 +33,96 @@ believe that when posting ideas or submitting code to an open-source project,
 it should be obvious and self-evident that any such contributions
 are made in the spirit of open collaborative development.
 
+Setup
+-----
+
+.. code-block:: bash
+
+    # Install Poetry, minimum version >=1.2 required
+    curl -sSL https://install.python-poetry.org | python -
+
+    # You may have to reinitialize your shell at this point.
+    source ~/.bashrc
+
+    # Activate in-project virtualenvs
+    poetry config virtualenvs.in-project true
+
+    # Add dynamic versioning plugin
+    poetry self add "poetry-dynamic-versioning[plugin]"
+
+    # Install dependencies
+    # (using "--with docs" if docs dependencies should be installed as well)
+    poetry install --with docs,server,dev
+
+    # Activate virtual environment
+    poetry shell
+    
+
+Update poetry
+^^^^^^^^^^^^^
+
+Developing pyam requires poetry ``>= 1.2``.
+
+If you already have a previous version of poetry installed you will need to update. The
+first step is removing the old poetry version:
+
+.. code-block:: bash
+
+    curl -sSL https://install.python-poetry.org | python3 - --uninstall
+
+
+after that, the latest poetry version can be installed using:
+
+.. code-block:: bash
+
+    curl -sSL https://install.python-poetry.org | python3 -
+
+
+details can be found here in the poetry docs:
+https://python-poetry.org/docs/#installation.
+
+Resolve conflicts in poetry.lock
+--------------------------------
+
+When updating dependencies it can happen that a conflict between the current and the
+target poetry.lock file occurs. In this case the following steps should be taken to
+resolve the conflict.
+
+#. Do not attempt to manually resolve in the GitHub web interface.
+#. Instead checkout the target branch locally and merge into your branch:
+
+.. code-block:: bash
+
+    git checkout main
+    git pull origin main
+    git checkout my-branch
+    git merge main
+
+
+#. After the last step you'll have a merge conflict in poetry.lock.
+#. Instead of resolving the conflict, directly checkout the one from main and rewrite
+   it:
+
+.. code-block:: bash
+
+    # Get poetry.lock to look like it does in master
+    git checkout main poetry.lock
+    # Rewrite the lock file
+    poetry lock --no-update
+
+#. After that simply add poetry.lock to mark the conflict as resolved and commit to
+   finalize the merge:
+
+.. code-block:: bash
+
+    git add poetry.lock
+    git commit
+
+    # and most likely needed
+    poetry install
+
+(Taken from https://www.peterbe.com/plog/how-to-resolve-a-git-conflict-in-poetry.lock)
+
 .. _`pep8`: https://www.python.org/dev/peps/pep-0008/
 
 .. _`numpydoc docstring guide`: https://numpydoc.readthedocs.io/en/latest/format.html

diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
@@ -1,5 +1,12 @@
 # Next release
 
+## Dependency changes
+
+Bumped minimum version of pandas and numpy to fit **ixmp4**'s requirement.
+
+## Individual updates
+
+- [#827](https://github.com/IAMconsortium/pyam/pull/827) Migrate to poetry for project management
 - [#830](https://github.com/IAMconsortium/pyam/pull/830) Implement more consistent logging behavior with **ixmp4**
 - [#829](https://github.com/IAMconsortium/pyam/pull/829) Add a `pyam.iiasa.platforms()` function for a list of available platforms
 - [#826](https://github.com/IAMconsortium/pyam/pull/826) Add `read_ixmp4()` function and extend integration test

diff --git a/docs/conf.py b/docs/conf.py
@@ -327,10 +327,10 @@
     "pint": ("https://pint.readthedocs.io/en/stable", None),
     "matplotlib": ("https://matplotlib.org/stable/", None),
     "plotly": ("https://plotly.com/python-api-reference/", None),
-    "pandas_datareader": ("https://pandas-datareader.readthedocs.io/en/stable", None),
     "unfccc_di_api": ("https://unfccc-di-api.readthedocs.io/en/stable", None),
     "nomenclature": ("https://nomenclature-iamc.readthedocs.io/en/stable", None),
     "ixmp4": ("https://docs.ece.iiasa.ac.at/projects/ixmp4/en/stable", None),
+    "wbdata": ("https://wbdata.readthedocs.io/en/stable/", None),
 }
 
 # Set up the plotting gallery with plotly scraper