From a54c3b30a04b2aeb9f37a407ccb278e362b83f8b Mon Sep 17 00:00:00 2001 From: Al Rigazzi Date: Fri, 24 Jan 2025 15:24:51 +0100 Subject: [PATCH] Drop python 3.9 (#781) This PR drops support for Python 3.9. [ committed by @al-rigazzi ] [ reviewed by @ashao ] --- .github/workflows/release.yml | 2 +- .github/workflows/run_tests.yml | 9 +-- doc/changelog.md | 4 + doc/installation_instructions/basic.rst | 18 ++--- doc/installation_instructions/platform.rst | 5 -- .../platform/olcf-summit.rst | 75 ------------------- doc/smartsim_zoo.rst | 4 +- pyproject.toml | 2 +- setup.cfg | 3 +- smartsim/__init__.py | 4 +- smartsim/_core/_cli/scripts/dragon_install.py | 4 +- smartsim/_core/_install/buildenv.py | 2 +- tests/test_configs/telemetry/ensembles.json | 16 ++-- tests/test_dragon_installer.py | 16 +--- 14 files changed, 38 insertions(+), 126 deletions(-) delete mode 100644 doc/installation_instructions/platform/olcf-summit.rst diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e57994633d..984fee285f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -40,7 +40,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: '3.9' + python-version: '3.10' - name: Install build run: python -m pip install 'build>=1.2.2,<2' diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 4738692bfb..9ec3ecc295 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -58,10 +58,7 @@ jobs: os: [macos-14, ubuntu-22.04] # Operating systems compiler: [8] # GNU compiler version rai: [1.2.7] # Redis AI versions - py_v: ["3.9", "3.10", "3.11"] # Python versions - exclude: - - os: macos-14 - py_v: "3.9" + py_v: ["3.10", "3.11"] # Python versions steps: - uses: actions/checkout@v4 @@ -84,7 +81,7 @@ jobs: echo "$(brew --prefix)/opt/make/libexec/gnubin" >> $GITHUB_PATH - name: Build Singularity from source - if: contains( matrix.os, 'ubuntu' ) && matrix.py_v == 3.9 + if: contains( matrix.os, 'ubuntu' ) && matrix.py_v == 3.10 run: | sudo apt-get install -y libseccomp-dev pkg-config squashfs-tools cryptsetup curl git # wget build-essential echo 'export PATH=/usr/local/go/bin:$PATH' >> ~/.bashrc @@ -98,7 +95,7 @@ jobs: sudo make -C builddir install - name: singularity pull test container # This lets us time how long the pull takes - if: contains( matrix.os, 'ubuntu' ) && matrix.py_v == 3.9 + if: contains( matrix.os, 'ubuntu' ) && matrix.py_v == 3.10 run: singularity pull docker://alrigazzi/smartsim-testing # Note: The develop branch of smartredis is installed first to ensure that any tests that depend diff --git a/doc/changelog.md b/doc/changelog.md index 234d159528..9c5913fe7b 100644 --- a/doc/changelog.md +++ b/doc/changelog.md @@ -13,6 +13,7 @@ To be released at some point in the future Description +- Drop Python 3.9 support - Terminate LSF and LSB support - Implement workaround for Tensorflow that allows RedisAI to build with GCC-14 - Add instructions for installing SmartSim on PML's Scylla @@ -20,6 +21,9 @@ Description Detailed Notes +- Python 3.9 will not be supported anymore, the last stable version of SmartSim + with support for Python 3.9 will be 0.8. + ([SmartSim-PR781](https://github.com/CrayLabs/SmartSim/pull/781)) - After the supercomputer Summit was decommissioned, a decision was made to terminate SmartSim's support of the LSF launcher and LSB scheduler. If this impacts your work, please contact us. diff --git a/doc/installation_instructions/basic.rst b/doc/installation_instructions/basic.rst index 226ccb0854..57faeab8d1 100644 --- a/doc/installation_instructions/basic.rst +++ b/doc/installation_instructions/basic.rst @@ -20,7 +20,7 @@ Basic The base prerequisites to install SmartSim and SmartRedis wtih CPU-only support are: - - Python 3.9-3.11 + - Python 3.10-3.11 - Pip - Cmake 3.13.x (or later) - C compiler @@ -77,7 +77,7 @@ Linux - Torch - Tensorflow - ONNX Runtime - * - 3.9-3.11 + * - 3.10-3.11 - 2.3.1 - 2.14.1 - 1.17.3 @@ -98,7 +98,7 @@ Linux - Torch - Tensorflow - ONNX Runtime - * - 3.9-3.11 + * - 3.10-3.11 - 2.3.1 - 2.17 - 1.17.3 @@ -114,7 +114,7 @@ Linux - Torch - Tensorflow - ONNX Runtime - * - 3.9-3.11 + * - 3.10-3.11 - 2.4.1 - N/A - N/A @@ -130,7 +130,7 @@ Linux - Torch - Tensorflow - ONNX Runtime - * - 3.9-3.11 + * - 3.10-3.11 - 2.4.0 - 2.15 - 1.17.3 @@ -151,7 +151,7 @@ MacOSX - Torch - Tensorflow - ONNX Runtime - * - 3.9-3.11 + * - 3.10-3.11 - 2.4.0 - 2.17 - 1.17.3 @@ -167,7 +167,7 @@ MacOSX - Torch - Tensorflow - ONNX Runtime - * - 3.9-3.11 + * - 3.10-3.11 - 2.2.0 - 2.15 - 1.17.3 @@ -328,9 +328,9 @@ SmartSim does. * - Platform - Python Versions * - MacOS - - 3.9 - 3.11 + - 3.10 - 3.11 * - Linux - - 3.9 - 3.11 + - 3.10 - 3.11 The Python client for SmartRedis is installed through ``pip`` as follows: diff --git a/doc/installation_instructions/platform.rst b/doc/installation_instructions/platform.rst index c1eb51df1a..06d8a32c36 100644 --- a/doc/installation_instructions/platform.rst +++ b/doc/installation_instructions/platform.rst @@ -18,13 +18,8 @@ that SmartSim may be used on. .. include:: platform/ncar-cheyenne.rst -.. include:: platform/olcf-summit.rst - .. include:: platform/pml-scylla.rst .. _site_installation: .. include:: site-install.rst - - - diff --git a/doc/installation_instructions/platform/olcf-summit.rst b/doc/installation_instructions/platform/olcf-summit.rst deleted file mode 100644 index 07be24eec7..0000000000 --- a/doc/installation_instructions/platform/olcf-summit.rst +++ /dev/null @@ -1,75 +0,0 @@ - -Summit at OLCF -============== - -Since SmartSim does not have a built PowerPC build, the build steps for an IBM -system are slightly different than other systems. - -Luckily for us, a conda channel with all relevant packages is maintained as part -of the `OpenCE `_ -initiative. Users can follow these instructions to get a working SmartSim build -with PyTorch and TensorFlow for GPU on Summit. Note that SmartSim and SmartRedis -will be downloaded to the working directory from which these instructions are executed. - -Note that the available PyTorch version (1.10.2) does not match -the one expected by RedisAI 1.2.7 (1.11): it is still compatible and should -work, but please open an issue on SmartSim's GitHub repo if you run -into problems. - -.. code-block:: bash - - # setup Python and build environment - export ENV_NAME=smartsim-0.8.0 - git clone https://github.com/CrayLabs/SmartRedis.git smartredis - git clone https://github.com/CrayLabs/SmartSim.git smartsim - conda config --prepend channels https://ftp.osuosl.org/pub/open-ce/1.6.1/ - conda create --name $ENV_NAME -y python=3.9 \ - git-lfs \ - cmake \ - make \ - cudnn=8.1.1_11.2 \ - cudatoolkit=11.2.2 \ - tensorflow=2.8.1 \ - libtensorflow \ - pytorch=1.10.2 \ - torchvision=0.11.3 - conda activate $ENV_NAME - export CC=$(which gcc) - export CXX=$(which g++) - export LDFLAGS="$LDFLAGS -pthread" - export CUDNN_LIBRARY=/ccs/home/$USER/.conda/envs/$ENV_NAME/lib/ - export CUDNN_INCLUDE_DIR=/ccs/home/$USER/.conda/envs/$ENV_NAME/include/ - module load cuda/11.4.2 - export LD_LIBRARY_PATH=$CUDNN_LIBRARY:$LD_LIBRARY_PATH:/ccs/home/$USER/.conda/envs/$ENV_NAME/lib/python3.9/site-packages/torch/lib - module load gcc/9.3.0 - module unload xalt - # clone SmartRedis and build - pushd smartredis - make lib && pip install . - popd - - # clone SmartSim and build - pushd smartsim - pip install . - - # install PyTorch and TensorFlow backend for the Orchestrator database. - export Torch_DIR=/ccs/home/$USER/.conda/envs/$ENV_NAME/lib/python3.9/site-packages/torch/share/cmake/Torch/ - export CFLAGS="$CFLAGS -I/ccs/home/$USER/.conda/envs/$ENV_NAME/lib/python3.9/site-packages/tensorflow/include" - export SMARTSIM_REDISAI=1.2.7 - export Tensorflow_BUILD_DIR=/ccs/home/$USER/.conda/envs/$ENV_NAME/lib/python3.9/site-packages/tensorflow/ - smart build --device=gpu --torch_dir $Torch_DIR --libtensorflow_dir $Tensorflow_BUILD_DIR -v - - # Show LD_LIBRARY_PATH for future reference - echo "SmartSim installation is complete, LD_LIBRARY_PATH=$LD_LIBRARY_PATH" - -When executing SmartSim, if you want to use the PyTorch and TensorFlow backends -in the orchestrator, you will need to set up the same environment used at build -time: - -.. code-block:: bash - - module load cuda/11.4.2 - export CUDNN_LIBRARY=/ccs/home/$USER/.conda/envs/$ENV_NAME/lib/ - export LD_LIBRARY_PATH=/ccs/home/$USER/.conda/envs/smartsim/lib/python3.8/site-packages/torch/lib/:$LD_LIBRARY_PATH:$CUDNN_LIBRARY - module load gcc/9.3.0 - module unload xalt diff --git a/doc/smartsim_zoo.rst b/doc/smartsim_zoo.rst index 1ce803bf6a..e1bf7fc39f 100644 --- a/doc/smartsim_zoo.rst +++ b/doc/smartsim_zoo.rst @@ -83,12 +83,12 @@ can find a more detailed description of each example in the `SmartSim Zoo - National Center for Atmospheric Research (NCAR) * - Cheyenne - National Center for Atmospheric Research (NCAR) - * - Summit - - Oak Ridge National Lab * - Theta - Argonne National Lab * - ThetaGPU - Argonne National Lab + * - Scylla + - Plymouth Marine Laboratory .. list-table:: Current CrayLabs Collaborations :widths: 100 150 100 diff --git a/pyproject.toml b/pyproject.toml index 62df92f0c9..307f85e9e7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ build-backend = "setuptools.build_meta" [tool.black] line-length = 88 -target-version = ['py39', 'py310', 'py311'] +target-version = ['py310', 'py311'] exclude = ''' ( | \.egg diff --git a/setup.cfg b/setup.cfg index 1ea8d2518b..46064e9be8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -42,7 +42,6 @@ contact_email = craylabs@hpe.com license = BSD 2-Clause License keywords = scientific, ai, workflow, hpc, analysis classifiers = - Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 License :: OSI Approved :: BSD License @@ -52,7 +51,7 @@ classifiers = [options] packages = find: include_package_data = True -python_requires = >=3.9,<3.12 +python_requires = >=3.10,<3.12 [options.packages.find] include = diff --git a/smartsim/__init__.py b/smartsim/__init__.py index 5e24097a57..8431a525ac 100644 --- a/smartsim/__init__.py +++ b/smartsim/__init__.py @@ -30,8 +30,8 @@ # pylint: disable-next=useless-import-alias from .version import __version__ as __version__ -if sys.version_info < (3, 9): # pragma: no cover - sys.exit("Python 3.9 or greater must be used with SmartSim.") +if sys.version_info < (3, 10): # pragma: no cover + sys.exit("Python 3.10 or greater must be used with SmartSim.") # Main API module # pylint: disable=wrong-import-position diff --git a/smartsim/_core/_cli/scripts/dragon_install.py b/smartsim/_core/_cli/scripts/dragon_install.py index 8028b8ecfd..cfdc51a9bb 100644 --- a/smartsim/_core/_cli/scripts/dragon_install.py +++ b/smartsim/_core/_cli/scripts/dragon_install.py @@ -114,8 +114,8 @@ def filter_assets(assets: t.Collection[GitReleaseAsset]) -> t.Optional[GitReleas :param assets: The collection of dragon release assets to filter :returns: An asset meeting platform & version filtering requirements""" # Expect cray & non-cray assets that require a filter, e.g. - # 'dragon-0.8-py3.9.4.1-bafaa887f.tar.gz', - # 'dragon-0.8-py3.9.4.1-CRAYEX-ac132fe95.tar.gz' + # 'dragon-0.8-py3.10.4.1-bafaa887f.tar.gz', + # 'dragon-0.8-py3.10.4.1-CRAYEX-ac132fe95.tar.gz' asset = next( ( asset diff --git a/smartsim/_core/_install/buildenv.py b/smartsim/_core/_install/buildenv.py index bff421b129..986c65203d 100644 --- a/smartsim/_core/_install/buildenv.py +++ b/smartsim/_core/_install/buildenv.py @@ -154,7 +154,7 @@ class Versioner: """ # compatible Python version - PYTHON_MIN = Version_("3.9.0") + PYTHON_MIN = Version_("3.10.0") # Versions SMARTSIM = Version_(get_env("SMARTSIM_VERSION", "0.8.0")) diff --git a/tests/test_configs/telemetry/ensembles.json b/tests/test_configs/telemetry/ensembles.json index 67e53ca096..632bf84068 100644 --- a/tests/test_configs/telemetry/ensembles.json +++ b/tests/test_configs/telemetry/ensembles.json @@ -41,7 +41,7 @@ ], "run_settings": { "exe": [ - "/home/someuser/.pyenv/versions/3.9.16/envs/ss/bin/python" + "/home/someuser/.pyenv/versions/3.10.16/envs/ss/bin/python" ], "run_command": null, "run_args": {} @@ -77,7 +77,7 @@ ], "run_settings": { "exe": [ - "/home/someuser/.pyenv/versions/3.9.16/envs/ss/bin/python" + "/home/someuser/.pyenv/versions/3.10.16/envs/ss/bin/python" ], "run_command": null, "run_args": {} @@ -113,7 +113,7 @@ ], "run_settings": { "exe": [ - "/home/someuser/.pyenv/versions/3.9.16/envs/ss/bin/python" + "/home/someuser/.pyenv/versions/3.10.16/envs/ss/bin/python" ], "run_command": null, "run_args": {} @@ -149,7 +149,7 @@ ], "run_settings": { "exe": [ - "/home/someuser/.pyenv/versions/3.9.16/envs/ss/bin/python" + "/home/someuser/.pyenv/versions/3.10.16/envs/ss/bin/python" ], "run_command": null, "run_args": {} @@ -185,7 +185,7 @@ ], "run_settings": { "exe": [ - "/home/someuser/.pyenv/versions/3.9.16/envs/ss/bin/python" + "/home/someuser/.pyenv/versions/3.10.16/envs/ss/bin/python" ], "run_command": null, "run_args": {} @@ -221,7 +221,7 @@ ], "run_settings": { "exe": [ - "/home/someuser/.pyenv/versions/3.9.16/envs/ss/bin/python" + "/home/someuser/.pyenv/versions/3.10.16/envs/ss/bin/python" ], "run_command": null, "run_args": {} @@ -257,7 +257,7 @@ ], "run_settings": { "exe": [ - "/home/someuser/.pyenv/versions/3.9.16/envs/ss/bin/python" + "/home/someuser/.pyenv/versions/3.10.16/envs/ss/bin/python" ], "run_command": null, "run_args": {} @@ -293,7 +293,7 @@ ], "run_settings": { "exe": [ - "/home/someuser/.pyenv/versions/3.9.16/envs/ss/bin/python" + "/home/someuser/.pyenv/versions/3.10.16/envs/ss/bin/python" ], "run_command": null, "run_args": {} diff --git a/tests/test_dragon_installer.py b/tests/test_dragon_installer.py index b23a1a7ef0..1d967d14e1 100644 --- a/tests/test_dragon_installer.py +++ b/tests/test_dragon_installer.py @@ -50,7 +50,7 @@ pytestmark = pytest.mark.group_a -mock_archive_name = "dragon-0.8-py3.9.4.1-CRAYEX-ac132fe95.tar.gz" +mock_archive_name = "dragon-0.8-py3.10.4.1-CRAYEX-ac132fe95.tar.gz" _git_attr = namedtuple("_git_attr", "value") @@ -102,7 +102,7 @@ def test_assets(monkeypatch: pytest.MonkeyPatch) -> t.Dict[str, GitReleaseAsset] assets: t.List[GitReleaseAsset] = [] mock_archive_name_tpl = "{}-{}.4.1-{}ac132fe95.tar.gz" - for python_version in ["py3.9", "py3.10", "py3.11"]: + for python_version in ["py3.10", "py3.11"]: for dragon_version in ["dragon-0.8", "dragon-0.9", "dragon-0.10"]: for platform in ["", "CRAYEX-"]: @@ -187,29 +187,21 @@ def test_retrieve_cached( @pytest.mark.parametrize( "dragon_pin,pyv,is_found,is_crayex", [ - pytest.param("0.8", "py3.8", False, False, id="0.8,python 3.8"), - pytest.param("0.8", "py3.9", True, False, id="0.8,python 3.9"), pytest.param("0.8", "py3.10", True, False, id="0.8,python 3.10"), pytest.param("0.8", "py3.11", True, False, id="0.8,python 3.11"), pytest.param("0.8", "py3.12", False, False, id="0.8,python 3.12"), - pytest.param("0.8", "py3.8", False, True, id="0.8,python 3.8,CrayEX"), - pytest.param("0.8", "py3.9", True, True, id="0.8,python 3.9,CrayEX"), pytest.param("0.8", "py3.10", True, True, id="0.8,python 3.10,CrayEX"), pytest.param("0.8", "py3.11", True, True, id="0.8,python 3.11,CrayEX"), pytest.param("0.8", "py3.12", False, True, id="0.8,python 3.12,CrayEX"), - pytest.param("0.9", "py3.8", False, False, id="0.9,python 3.8"), - pytest.param("0.9", "py3.9", True, False, id="0.9,python 3.9"), pytest.param("0.9", "py3.10", True, False, id="0.9,python 3.10"), pytest.param("0.9", "py3.11", True, False, id="0.9,python 3.11"), pytest.param("0.9", "py3.12", False, False, id="0.9,python 3.12"), - pytest.param("0.9", "py3.8", False, True, id="0.9,python 3.8,CrayEX"), - pytest.param("0.9", "py3.9", True, True, id="0.9,python 3.9,CrayEX"), pytest.param("0.9", "py3.10", True, True, id="0.9,python 3.10,CrayEX"), pytest.param("0.9", "py3.11", True, True, id="0.9,python 3.11,CrayEX"), pytest.param("0.9", "py3.12", False, True, id="0.9,python 3.12,CrayEX"), # add a couple variants for a dragon version that isn't in the asset list - pytest.param("0.7", "py3.9", False, False, id="0.7,python 3.9"), - pytest.param("0.7", "py3.9", False, True, id="0.7,python 3.9,CrayEX"), + pytest.param("0.7", "py3.10", False, False, id="0.7,python 3.10"), + pytest.param("0.7", "py3.10", False, True, id="0.7,python 3.10,CrayEX"), ], ) def test_retrieve_asset_info(