From b1b59ca65d23e7504be0813d78bf46949fffac46 Mon Sep 17 00:00:00 2001 From: daniel Date: Tue, 5 Sep 2023 16:44:51 +0800 Subject: [PATCH] fix pyyaml dependecies clashing in kafka-setup with reference to pr #8435 --- docker/kafka-setup/Dockerfile | 6 ++++-- docker/quickstart/generate_and_compare.sh | 1 + docker/quickstart/requirements.txt | 2 +- metadata-ingestion-modules/airflow-plugin/build.gradle | 4 ++++ 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/docker/kafka-setup/Dockerfile b/docker/kafka-setup/Dockerfile index e379ae4385246..bb0a7aac4737c 100644 --- a/docker/kafka-setup/Dockerfile +++ b/docker/kafka-setup/Dockerfile @@ -17,7 +17,7 @@ ENV SCALA_VERSION 2.13 ENV CUB_CLASSPATH='"/usr/share/java/cp-base-new/*"' # Confluent Docker Utils Version (Namely the tag or branch to grab from git to install) -ARG PYTHON_CONFLUENT_DOCKER_UTILS_VERSION="v0.0.58" +ARG PYTHON_CONFLUENT_DOCKER_UTILS_VERSION="v0.0.60" # This can be overriden for an offline/air-gapped builds ARG PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC="git+https://github.com/confluentinc/confluent-docker-utils@${PYTHON_CONFLUENT_DOCKER_UTILS_VERSION}" @@ -36,7 +36,9 @@ RUN mkdir -p /opt \ && adduser -DH -s /sbin/nologin kafka \ && chown -R kafka: /opt/kafka \ && echo "===> Installing python packages ..." \ - && pip install --no-cache-dir jinja2 requests \ + && pip install --no-cache-dir --upgrade pip wheel setuptools \ + && pip install jinja2 requests \ + && pip install "Cython<3.0" "PyYAML<6" --no-build-isolation \ && pip install --prefer-binary --prefix=/usr/local --upgrade "${PYTHON_CONFLUENT_DOCKER_UTILS_INSTALL_SPEC}" \ && rm -rf /tmp/* \ && apk del --purge .build-deps diff --git a/docker/quickstart/generate_and_compare.sh b/docker/quickstart/generate_and_compare.sh index e34abeb9820c6..d568eb3a4c246 100755 --- a/docker/quickstart/generate_and_compare.sh +++ b/docker/quickstart/generate_and_compare.sh @@ -8,5 +8,6 @@ set -euxo pipefail python3 -m venv venv source venv/bin/activate +pip install --upgrade pip wheel setuptools pip install -r requirements.txt python generate_docker_quickstart.py check-all diff --git a/docker/quickstart/requirements.txt b/docker/quickstart/requirements.txt index a20e96afc8582..539241331120c 100644 --- a/docker/quickstart/requirements.txt +++ b/docker/quickstart/requirements.txt @@ -1,3 +1,3 @@ -PyYAML==5.4.1 +PyYAML==6.0 python-dotenv==0.17.0 click diff --git a/metadata-ingestion-modules/airflow-plugin/build.gradle b/metadata-ingestion-modules/airflow-plugin/build.gradle index 9ab590b1560b3..d895e29229c41 100644 --- a/metadata-ingestion-modules/airflow-plugin/build.gradle +++ b/metadata-ingestion-modules/airflow-plugin/build.gradle @@ -22,6 +22,10 @@ task environmentSetup(type: Exec, dependsOn: checkPythonVersion) { task installPackage(type: Exec, dependsOn: environmentSetup) { inputs.file file('setup.py') outputs.dir("${venv_name}") + // Workaround for https://github.com/yaml/pyyaml/issues/601. + // See https://github.com/yaml/pyyaml/issues/601#issuecomment-1638509577. + // and https://github.com/datahub-project/datahub/pull/8435. + commandLine 'bash', '-x', '-c', "${pip_install_command} install 'Cython<3.0' 'PyYAML<6' --no-build-isolation" commandLine 'bash', '-x', '-c', "${pip_install_command} -e ." }