From bfd15f7390b3409a659978326c5db1a18237eee5 Mon Sep 17 00:00:00 2001 From: amanda Date: Wed, 27 Jan 2021 23:35:22 +0000 Subject: [PATCH 1/8] Initial updates on build for updated pip/venv --- packages/st2/Makefile | 6 ++++-- packages/st2/component.makefile | 4 ++-- rake/build/upload_checkout.rake | 2 +- rpmspec/package_venv.spec | 7 ++++--- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/packages/st2/Makefile b/packages/st2/Makefile index 66c4590e..378eb291 100644 --- a/packages/st2/Makefile +++ b/packages/st2/Makefile @@ -117,6 +117,8 @@ populate_version: .stamp-populate_version touch $@ requirements: + $(PYTHON_BINARY) -c "import pip; print(pip.__version__)" + echo $(PYTHON_BINARY) $(PYTHON_BINARY) ../scripts/fixate-requirements.py -s ../st2*/in-requirements.txt ../contrib/runners/*/in-requirements.txt -f ../fixed-requirements.txt cat requirements.txt @@ -130,8 +132,8 @@ wheelhouse: .build-runners .stamp-wheelhouse .stamp-wheelhouse: | populate_version requirements inject-deps cat requirements.txt # Try to install wheels 2x in case the first one fails - $(PIP_BINARY) wheel --wheel-dir=$(WHEELDIR) --find-links=$(WHEELDIR) -r requirements.txt || \ - $(PIP_BINARY) wheel --wheel-dir=$(WHEELDIR) --find-links=$(WHEELDIR) -r requirements.txt + $(PIP_BINARY) --use-deprecated=legacy-resolver wheel --wheel-dir=$(WHEELDIR) --find-links=$(WHEELDIR) -r requirements.txt || \ + $(PIP_BINARY) --use-deprecated=legacy-resolver wheel --wheel-dir=$(WHEELDIR) --find-links=$(WHEELDIR) -r requirements.txt touch $@ .build-runners: diff --git a/packages/st2/component.makefile b/packages/st2/component.makefile index 7a752f60..ef62643c 100644 --- a/packages/st2/component.makefile +++ b/packages/st2/component.makefile @@ -74,8 +74,8 @@ wheelhouse: .stamp-wheelhouse # Install wheels into shared location cat requirements.txt # Try to install wheels 2x in case the first one fails - $(PIP_BINARY) wheel --wheel-dir=$(WHEELDIR) --find-links=$(WHEELDIR) -r requirements.txt || \ - $(PIP_BINARY) wheel --wheel-dir=$(WHEELDIR) --find-links=$(WHEELDIR) -r requirements.txt + $(PIP_BINARY) --use-deprecated=legacy-resolver wheel --wheel-dir=$(WHEELDIR) --find-links=$(WHEELDIR) -r requirements.txt || \ + $(PIP_BINARY) --use-deprecated=legacy-resolver wheel --wheel-dir=$(WHEELDIR) --find-links=$(WHEELDIR) -r requirements.txt touch $@ bdist_wheel: .stamp-bdist_wheel diff --git a/rake/build/upload_checkout.rake b/rake/build/upload_checkout.rake index a3247403..86649418 100644 --- a/rake/build/upload_checkout.rake +++ b/rake/build/upload_checkout.rake @@ -53,7 +53,7 @@ namespace :upload do execute :mkdir, '-p $ARTIFACT_DIR' within opts.basedir do if opts.checkout == 1 - execute :git, :clone, '--depth 1 -b $GITREV $GITURL $GITDIR' + execute :git, :clone, '--depth 1 -b amanda_tmp_virtualenv $GITURL $GITDIR' end end end diff --git a/rpmspec/package_venv.spec b/rpmspec/package_venv.spec index 968ab381..62c1af7e 100644 --- a/rpmspec/package_venv.spec +++ b/rpmspec/package_venv.spec @@ -8,10 +8,10 @@ %define venv_bin %{venv_dir}/bin %define venv_python %{venv_bin}/python3 -%define pin_pip %{venv_python} %{venv_bin}/pip3 install pip==19.1.1 +%define pin_pip %{venv_python} %{venv_bin}/pip3 install pip==20.3.4 %define install_venvctrl python3 -m pip install venvctrl %if 0%{?rhel} == 8 -%define install_crypto %{venv_python} %{venv_bin}/pip install cryptography==2.8 --no-binary cryptography +%define install_crypto %{venv_python} %{venv_bin}/pip3 install cryptography==2.8 --no-binary cryptography %else %define install_crypto %{nil} %endif @@ -26,8 +26,9 @@ %define pip_install_venv \ virtualenv-3 -p python3 --no-download %{venv_dir} \ %{pin_pip} \ + %{venv_python} %{venv_bin}/pip3 --version \ %{install_crypto} \ - %{venv_pip} -r requirements.txt \ + %{venv_pip} --use-deprecated=legacy-resolver -r requirements.txt \ %{venv_pip} . \ %{install_venvctrl} \ venvctrl-relocate --source=%{venv_dir} --destination=/%{venv_install_dir} \ From 3d2f4706b17fc408590ca1fd6b3b232c4fcbde50 Mon Sep 17 00:00:00 2001 From: amanda Date: Thu, 28 Jan 2021 13:20:31 +0000 Subject: [PATCH 2/8] Update accidentally checked in file --- rake/build/upload_checkout.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rake/build/upload_checkout.rake b/rake/build/upload_checkout.rake index 86649418..a3247403 100644 --- a/rake/build/upload_checkout.rake +++ b/rake/build/upload_checkout.rake @@ -53,7 +53,7 @@ namespace :upload do execute :mkdir, '-p $ARTIFACT_DIR' within opts.basedir do if opts.checkout == 1 - execute :git, :clone, '--depth 1 -b amanda_tmp_virtualenv $GITURL $GITDIR' + execute :git, :clone, '--depth 1 -b $GITREV $GITURL $GITDIR' end end end From c448cb0605465d257e00ab056399d3e238ab5218 Mon Sep 17 00:00:00 2001 From: amanda Date: Thu, 28 Jan 2021 21:35:37 +0000 Subject: [PATCH 3/8] Tidy-up --- packages/st2/Makefile | 2 -- rpmspec/package_venv.spec | 3 +-- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/st2/Makefile b/packages/st2/Makefile index 378eb291..f2a45446 100644 --- a/packages/st2/Makefile +++ b/packages/st2/Makefile @@ -117,8 +117,6 @@ populate_version: .stamp-populate_version touch $@ requirements: - $(PYTHON_BINARY) -c "import pip; print(pip.__version__)" - echo $(PYTHON_BINARY) $(PYTHON_BINARY) ../scripts/fixate-requirements.py -s ../st2*/in-requirements.txt ../contrib/runners/*/in-requirements.txt -f ../fixed-requirements.txt cat requirements.txt diff --git a/rpmspec/package_venv.spec b/rpmspec/package_venv.spec index 62c1af7e..b99eaf53 100644 --- a/rpmspec/package_venv.spec +++ b/rpmspec/package_venv.spec @@ -11,7 +11,7 @@ %define pin_pip %{venv_python} %{venv_bin}/pip3 install pip==20.3.4 %define install_venvctrl python3 -m pip install venvctrl %if 0%{?rhel} == 8 -%define install_crypto %{venv_python} %{venv_bin}/pip3 install cryptography==2.8 --no-binary cryptography +%define install_crypto %{venv_python} %{venv_bin}/pip install cryptography==2.8 --no-binary cryptography %else %define install_crypto %{nil} %endif @@ -26,7 +26,6 @@ %define pip_install_venv \ virtualenv-3 -p python3 --no-download %{venv_dir} \ %{pin_pip} \ - %{venv_python} %{venv_bin}/pip3 --version \ %{install_crypto} \ %{venv_pip} --use-deprecated=legacy-resolver -r requirements.txt \ %{venv_pip} . \ From 8f9eef1e5557680c6701f814291b73360e61c025 Mon Sep 17 00:00:00 2001 From: amanda Date: Fri, 29 Jan 2021 15:37:02 +0000 Subject: [PATCH 4/8] Downgrade pip to 20.3.3 --- rpmspec/package_venv.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rpmspec/package_venv.spec b/rpmspec/package_venv.spec index b99eaf53..8302359c 100644 --- a/rpmspec/package_venv.spec +++ b/rpmspec/package_venv.spec @@ -8,7 +8,7 @@ %define venv_bin %{venv_dir}/bin %define venv_python %{venv_bin}/python3 -%define pin_pip %{venv_python} %{venv_bin}/pip3 install pip==20.3.4 +%define pin_pip %{venv_python} %{venv_bin}/pip3 install pip==20.3.3 %define install_venvctrl python3 -m pip install venvctrl %if 0%{?rhel} == 8 %define install_crypto %{venv_python} %{venv_bin}/pip install cryptography==2.8 --no-binary cryptography From 76c9352ae516c33ec0a1e7db6f4259e29bba914a Mon Sep 17 00:00:00 2001 From: amanda Date: Mon, 1 Feb 2021 20:18:53 +0000 Subject: [PATCH 5/8] Set DH_VIRTUALENV_ARGUUMENTS --- packages/st2/debian/rules | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/st2/debian/rules b/packages/st2/debian/rules index a155d981..c08f73c2 100755 --- a/packages/st2/debian/rules +++ b/packages/st2/debian/rules @@ -67,7 +67,10 @@ override_dh_virtualenv: # # NB! Use '--no-download' arg for 'virtualenv' is required, # otherwise it downloads latest PIP version instead of bundled/pinned one. - dh_virtualenv --extra-virtualenv-arg='--no-download' \ + # nB. Need to set environment variable DH_VIRTUALENB_ARGUMENTS as + # otherwise --no-site-packages is added which is not supported in + # virtualenv >= 20.0 + DH_VIRTUALENV_ARGUMENTS="" dh_virtualenv --extra-virtualenv-arg='--no-download' \ --extra-pip-arg '--find-links=$(WHEELDIR)' \ --extra-pip-arg '--no-index' --no-test From 3218c0bce594ec34346e4f9553abdec6b4e82690 Mon Sep 17 00:00:00 2001 From: amanda Date: Fri, 12 Feb 2021 15:18:06 +0000 Subject: [PATCH 6/8] Use copies --- packages/st2/debian/rules | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/st2/debian/rules b/packages/st2/debian/rules index c08f73c2..626a5430 100755 --- a/packages/st2/debian/rules +++ b/packages/st2/debian/rules @@ -67,10 +67,10 @@ override_dh_virtualenv: # # NB! Use '--no-download' arg for 'virtualenv' is required, # otherwise it downloads latest PIP version instead of bundled/pinned one. - # nB. Need to set environment variable DH_VIRTUALENB_ARGUMENTS as - # otherwise --no-site-packages is added which is not supported in - # virtualenv >= 20.0 - DH_VIRTUALENV_ARGUMENTS="" dh_virtualenv --extra-virtualenv-arg='--no-download' \ + # NB! Use '--copies' else /opt/stackstorm/st2/bin/python is symlink to + # /usr/bin/pythonx rather than copy - and this breaks pack install + # which follows the path to calculate the location of the bin directory + dh_virtualenv --extra-virtualenv-arg='--copies' --extra-virtualenv-arg='--no-download' \ --extra-pip-arg '--find-links=$(WHEELDIR)' \ --extra-pip-arg '--no-index' --no-test From 2239eddcd2f361d6ee238bb1e83cd7468bc437f8 Mon Sep 17 00:00:00 2001 From: amanda Date: Thu, 11 Mar 2021 16:26:48 +0000 Subject: [PATCH 7/8] Update virtualenv args --- packages/st2/debian/rules | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/st2/debian/rules b/packages/st2/debian/rules index 1ba02cb1..74fc2fa8 100755 --- a/packages/st2/debian/rules +++ b/packages/st2/debian/rules @@ -70,9 +70,6 @@ override_dh_virtualenv: # NB! Use '--copies' else /opt/stackstorm/st2/bin/python is symlink to # /usr/bin/pythonx rather than copy - and this breaks pack install # which follows the path to calculate the location of the bin directory - # TODO: Determine if need force-pip-version on the branch as well... - # --force-pip-version is needed to ensure correct pip version is installed - # --force-pip-version='20.0.2' \ dh_virtualenv --extra-virtualenv-arg='--copies' --extra-virtualenv-arg='--no-download' \ --extra-pip-arg '--find-links=$(WHEELDIR)' \ --extra-pip-arg '--no-index' --no-test From 2a89445df3e5b93bc420373ff97dd069ab8ecb40 Mon Sep 17 00:00:00 2001 From: amanda Date: Fri, 12 Mar 2021 12:12:14 +0000 Subject: [PATCH 8/8] Add support for upgrade-pip-to flag --- packages/st2/debian/rules | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/st2/debian/rules b/packages/st2/debian/rules index 74fc2fa8..2ae29880 100755 --- a/packages/st2/debian/rules +++ b/packages/st2/debian/rules @@ -11,6 +11,7 @@ PATH = /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin WHEELDIR ?= /tmp/wheelhouse DH_VIRTUALENV_INSTALL_ROOT := /opt/stackstorm export DH_VIRTUALENV_INSTALL_ROOT +PIP_VERSION = 20.3.3 IS_SYSTEMD = $(shell command -v dh_systemd_enable > /dev/null 2>&1 && echo true) DEB_DISTRO := $(shell lsb_release -cs) @@ -71,6 +72,7 @@ override_dh_virtualenv: # /usr/bin/pythonx rather than copy - and this breaks pack install # which follows the path to calculate the location of the bin directory dh_virtualenv --extra-virtualenv-arg='--copies' --extra-virtualenv-arg='--no-download' \ + '--upgrade-pip-to=$(PIP_VERSION)' \ --extra-pip-arg '--find-links=$(WHEELDIR)' \ --extra-pip-arg '--no-index' --no-test