From 94b190e3d73c16d402c1c42529a18bd625c4ea5b Mon Sep 17 00:00:00 2001 From: Sean Rose <1994030+sean-rose@users.noreply.github.com> Date: Thu, 7 Mar 2024 14:01:07 -0800 Subject: [PATCH] feat: Upgrade Airflow from 2.7.3 to 2.8.2 (#1935) * Upgrade Airflow from 2.7.3 to 2.8.2. * Update CircleCI Docker orb from 2.2.0 to 2.5.0. * Update Ruff config for version 0.2. * fix dockerfile URL scheme to use https for gcloud CLI * Restore `google-auth` extra, with its built-in dependency on `authlib`. * Add missing Airflow state colors for "restarting", "deferred", and "removed". * Explain the difference between our configured Airflow state colors and the defaults. --------- Co-authored-by: mikaeld --- .circleci/config.yml | 2 +- Dockerfile | 2 +- config/airflow_local_settings.py | 7 +- constraints.txt | 853 ++++++++++++++++--------------- docker-compose.yml | 19 +- pyproject.toml | 4 +- requirements.in | 10 +- requirements.txt | 356 ++++++------- 8 files changed, 634 insertions(+), 619 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6e37dfb87..d5dc0ce62 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,7 +11,7 @@ version: 2.1 orbs: - docker: circleci/docker@2.2.0 + docker: circleci/docker@2.5.0 python: circleci/python@2.1.1 jobs: diff --git a/Dockerfile b/Dockerfile index 11f082434..a99b6eea0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM apache/airflow:slim-2.7.3-python3.10 +FROM apache/airflow:slim-2.8.2-python3.10 ARG PROJECT_DIR="/opt/airflow" diff --git a/config/airflow_local_settings.py b/config/airflow_local_settings.py index 57d9c5304..0e64b1949 100644 --- a/config/airflow_local_settings.py +++ b/config/airflow_local_settings.py @@ -1,11 +1,14 @@ STATE_COLORS = { "queued": "gray", "running": "lime", - "success": "#0000FF", + "success": "#0000FF", # Rather than "green". + "restarting": "violet", "failed": "red", "up_for_retry": "gold", "up_for_reschedule": "turquoise", "upstream_failed": "orange", - "skipped": "pink", + "skipped": "pink", # Rather than "hotpink". + "deferred": "mediumpurple", + "removed": "lightgrey", "scheduled": "tan", } diff --git a/constraints.txt b/constraints.txt index e8523041e..a94b4f9bc 100644 --- a/constraints.txt +++ b/constraints.txt @@ -1,6 +1,6 @@ # -# This constraints file was automatically generated on 2023-11-01T21:34:43Z -# via "eager-upgrade" mechanism of PIP. For the "v2-7-test" branch of Airflow. +# This constraints file was automatically generated on 2024-02-24T08:57:38.741333 +# via "eager-upgrade" mechanism of PIP. For the "v2-8-test" branch of Airflow. # This variant of constraints install uses the HEAD of the branch version for 'apache-airflow' but installs # the providers from PIP-released packages at the moment of the constraint generation. # @@ -21,218 +21,222 @@ # 1. Reproducible installation of airflow with selected providers (note constraints are used): # # pip install "apache-airflow[celery,cncf.kubernetes,google,amazon,snowflake]==X.Y.Z" \ -# --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-X.Y.Z/constraints-3.8.txt" +# --constraint \ +# "https://raw.githubusercontent.com/apache/airflow/constraints-X.Y.Z/constraints-3.10.txt" # # 2. Installing own dependencies that are potentially not matching the constraints (note constraints are not # used, and apache-airflow==X.Y.Z is used to make sure there is no accidental airflow upgrade/downgrade. # -# pip install "apache-airflow==X.Y.Z" "snowflake-connector-python[pandas]==2.9.0" +# pip install "apache-airflow==X.Y.Z" "snowflake-connector-python[pandas]=N.M.O" # -Authlib==1.2.1 -Babel==2.13.1 -ConfigUpdater==3.1.1 +Authlib==1.3.0 +Babel==2.14.0 +ConfigUpdater==3.2 Deprecated==1.2.14 -Flask-AppBuilder==4.3.6 +Flask-AppBuilder==4.3.11 Flask-Babel==2.0.0 Flask-Bcrypt==1.0.1 Flask-Caching==2.1.0 -Flask-JWT-Extended==4.5.3 -Flask-Limiter==3.5.0 +Flask-JWT-Extended==4.6.0 +Flask-Limiter==3.5.1 Flask-Login==0.6.3 Flask-SQLAlchemy==2.5.1 Flask-Session==0.5.0 Flask-WTF==1.2.1 Flask==2.2.5 -GitPython==3.1.40 -JPype1==1.4.1 +GitPython==3.1.42 +JPype1==1.5.0 JayDeBeApi==1.2.3 -Jinja2==3.1.2 +Jinja2==3.1.3 Js2Py==0.74 -Mako==1.2.4 -Markdown==3.5.1 -MarkupSafe==2.1.3 -PyGithub==2.1.1 +Mako==1.3.2 +Markdown==3.5.2 +MarkupSafe==2.1.5 +PyAthena==3.3.0 +PyGithub==2.2.0 PyHive==0.7.0 PyJWT==2.8.0 PyNaCl==1.5.0 PyYAML==6.0.1 -Pygments==2.16.1 -SQLAlchemy-JSONField==1.0.1.post0 +Pygments==2.17.2 +SQLAlchemy-JSONField==1.0.2 SQLAlchemy-Utils==0.41.1 -SQLAlchemy==1.4.50 +SQLAlchemy==1.4.51 SecretStorage==3.3.3 Sphinx==5.3.0 -WTForms==3.0.1 +WTForms==3.1.2 Werkzeug==2.2.3 adal==1.2.7 -adlfs==2023.10.0 -aiobotocore==2.7.0 +adlfs==2024.2.0 +aiobotocore==2.11.2 aiofiles==23.2.1 -aiohttp==3.8.6 +aiohttp==3.9.3 aioitertools==0.11.0 -aioresponses==0.7.4 +aioresponses==0.7.6 aiosignal==1.3.1 -alabaster==0.7.13 -alembic==1.12.1 -alibabacloud-adb20211201==1.0.0 +alabaster==0.7.16 +alembic==1.13.1 +alibabacloud-adb20211201==1.2.8 alibabacloud-credentials==0.3.2 alibabacloud-endpoint-util==0.0.3 alibabacloud-gateway-spi==0.0.1 alibabacloud-openapi-util==0.2.2 -alibabacloud-tea-openapi==0.3.7 +alibabacloud-tea-openapi==0.3.8 alibabacloud-tea-util==0.3.11 alibabacloud-tea-xml==0.0.2 -alibabacloud-tea==0.3.3 +alibabacloud-tea==0.3.5 aliyun-python-sdk-core==2.14.0 aliyun-python-sdk-kms==2.16.2 -amqp==5.1.1 -analytics-python==1.4.post1 +amqp==5.2.0 +analytics-python==1.2.9 annotated-types==0.6.0 -ansiwrap==0.8.4 anyascii==0.3.2 -anyio==4.0.0 -apache-airflow-providers-airbyte==3.4.0 -apache-airflow-providers-alibaba==2.6.0 -apache-airflow-providers-amazon==8.10.0 -apache-airflow-providers-apache-beam==5.3.0 -apache-airflow-providers-apache-cassandra==3.3.0 -apache-airflow-providers-apache-drill==2.5.0 -apache-airflow-providers-apache-druid==3.6.0 -apache-airflow-providers-apache-flink==1.2.0 -apache-airflow-providers-apache-hdfs==4.2.0 -apache-airflow-providers-apache-hive==6.2.0 -apache-airflow-providers-apache-impala==1.2.0 -apache-airflow-providers-apache-kafka==1.2.0 -apache-airflow-providers-apache-kylin==3.3.0 -apache-airflow-providers-apache-livy==3.6.0 -apache-airflow-providers-apache-pig==4.2.0 -apache-airflow-providers-apache-pinot==4.2.0 -apache-airflow-providers-apache-spark==4.3.0 -apache-airflow-providers-apache-sqoop==4.1.0 -apache-airflow-providers-apprise==1.1.0 -apache-airflow-providers-arangodb==2.3.0 -apache-airflow-providers-asana==2.3.0 -apache-airflow-providers-atlassian-jira==2.2.0 -apache-airflow-providers-celery==3.4.1 -apache-airflow-providers-cloudant==3.3.0 -apache-airflow-providers-cncf-kubernetes==7.8.0 -apache-airflow-providers-common-sql==1.8.0 -apache-airflow-providers-daskexecutor==1.1.0 -apache-airflow-providers-databricks==4.7.0 -apache-airflow-providers-datadog==3.4.0 -apache-airflow-providers-dbt-cloud==3.4.0 -apache-airflow-providers-dingding==3.3.0 -apache-airflow-providers-discord==3.4.0 -apache-airflow-providers-docker==3.8.0 -apache-airflow-providers-elasticsearch==5.1.0 -apache-airflow-providers-exasol==4.3.0 -apache-airflow-providers-facebook==3.3.0 -apache-airflow-providers-ftp==3.6.0 -apache-airflow-providers-github==2.4.0 -apache-airflow-providers-google==10.11.0 -apache-airflow-providers-grpc==3.3.0 -apache-airflow-providers-hashicorp==3.5.0 -apache-airflow-providers-http==4.6.0 -apache-airflow-providers-imap==3.4.0 -apache-airflow-providers-influxdb==2.3.0 -apache-airflow-providers-jdbc==4.1.0 -apache-airflow-providers-jenkins==3.4.0 -apache-airflow-providers-microsoft-azure==8.1.0 -apache-airflow-providers-microsoft-mssql==3.5.0 -apache-airflow-providers-microsoft-psrp==2.4.0 -apache-airflow-providers-microsoft-winrm==3.3.0 -apache-airflow-providers-mongo==3.4.0 -apache-airflow-providers-mysql==5.4.0 -apache-airflow-providers-neo4j==3.4.0 -apache-airflow-providers-odbc==4.1.0 -apache-airflow-providers-openfaas==3.3.0 -apache-airflow-providers-openlineage==1.2.0 -apache-airflow-providers-opensearch==1.0.0 -apache-airflow-providers-opsgenie==5.2.0 -apache-airflow-providers-oracle==3.8.0 -apache-airflow-providers-pagerduty==3.4.0 -apache-airflow-providers-papermill==3.4.0 -apache-airflow-providers-plexus==3.3.0 -apache-airflow-providers-postgres==5.7.1 -apache-airflow-providers-presto==5.2.1 -apache-airflow-providers-redis==3.4.0 -apache-airflow-providers-salesforce==5.5.0 -apache-airflow-providers-samba==4.3.0 -apache-airflow-providers-segment==3.3.0 -apache-airflow-providers-sendgrid==3.3.0 -apache-airflow-providers-sftp==4.7.0 -apache-airflow-providers-singularity==3.3.0 -apache-airflow-providers-slack==8.3.0 -apache-airflow-providers-smtp==1.4.1 -apache-airflow-providers-snowflake==5.1.0 -apache-airflow-providers-sqlite==3.5.0 -apache-airflow-providers-ssh==3.8.1 -apache-airflow-providers-tableau==4.3.0 -apache-airflow-providers-tabular==1.3.0 -apache-airflow-providers-telegram==4.2.0 -apache-airflow-providers-trino==5.4.0 -apache-airflow-providers-vertica==3.6.0 -apache-airflow-providers-zendesk==4.4.0 -apache-beam==2.51.0 -apispec==6.3.0 -apprise==1.6.0 -argcomplete==3.1.3 -arrow==1.3.0 +anyio==4.3.0 +apache-airflow-providers-airbyte==3.6.0 +apache-airflow-providers-alibaba==2.7.2 +apache-airflow-providers-amazon==8.18.0 +apache-airflow-providers-apache-beam==5.6.1 +apache-airflow-providers-apache-cassandra==3.4.1 +apache-airflow-providers-apache-drill==2.6.1 +apache-airflow-providers-apache-druid==3.8.1 +apache-airflow-providers-apache-flink==1.3.0 +apache-airflow-providers-apache-hdfs==4.3.2 +apache-airflow-providers-apache-hive==7.0.0 +apache-airflow-providers-apache-impala==1.3.0 +apache-airflow-providers-apache-kafka==1.3.1 +apache-airflow-providers-apache-kylin==3.5.0 +apache-airflow-providers-apache-livy==3.7.2 +apache-airflow-providers-apache-pig==4.3.0 +apache-airflow-providers-apache-pinot==4.3.0 +apache-airflow-providers-apache-spark==4.7.1 +apache-airflow-providers-apprise==1.2.2 +apache-airflow-providers-arangodb==2.4.1 +apache-airflow-providers-asana==2.4.1 +apache-airflow-providers-atlassian-jira==2.5.1 +apache-airflow-providers-celery==3.6.0 +apache-airflow-providers-cloudant==3.4.1 +apache-airflow-providers-cncf-kubernetes==8.0.0 +apache-airflow-providers-cohere==1.1.2 +apache-airflow-providers-common-io==1.3.0 +apache-airflow-providers-common-sql==1.11.0 +apache-airflow-providers-databricks==6.2.0 +apache-airflow-providers-datadog==3.5.1 +apache-airflow-providers-dbt-cloud==3.6.1 +apache-airflow-providers-dingding==3.4.0 +apache-airflow-providers-discord==3.6.0 +apache-airflow-providers-docker==3.9.1 +apache-airflow-providers-elasticsearch==5.3.3 +apache-airflow-providers-exasol==4.4.2 +apache-airflow-providers-facebook==3.4.0 +apache-airflow-providers-ftp==3.7.0 +apache-airflow-providers-github==2.5.1 +apache-airflow-providers-google==10.15.0 +apache-airflow-providers-grpc==3.4.1 +apache-airflow-providers-hashicorp==3.6.3 +apache-airflow-providers-http==4.9.1 +apache-airflow-providers-imap==3.5.0 +apache-airflow-providers-influxdb==2.4.1 +apache-airflow-providers-jdbc==4.2.2 +apache-airflow-providers-jenkins==3.5.1 +apache-airflow-providers-microsoft-azure==9.0.0 +apache-airflow-providers-microsoft-mssql==3.6.1 +apache-airflow-providers-microsoft-psrp==2.5.0 +apache-airflow-providers-microsoft-winrm==3.4.0 +apache-airflow-providers-mongo==4.0.0 +apache-airflow-providers-mysql==5.5.3 +apache-airflow-providers-neo4j==3.5.0 +apache-airflow-providers-odbc==4.4.1 +apache-airflow-providers-openai==1.1.0 +apache-airflow-providers-openfaas==3.4.0 +apache-airflow-providers-openlineage==1.5.0 +apache-airflow-providers-opensearch==1.1.1 +apache-airflow-providers-opsgenie==5.5.0 +apache-airflow-providers-oracle==3.9.2 +apache-airflow-providers-pagerduty==3.6.2 +apache-airflow-providers-papermill==3.6.1 +apache-airflow-providers-pgvector==1.1.0 +apache-airflow-providers-pinecone==1.1.2 +apache-airflow-providers-postgres==5.10.1 +apache-airflow-providers-presto==5.4.1 +apache-airflow-providers-redis==3.6.0 +apache-airflow-providers-salesforce==5.6.2 +apache-airflow-providers-samba==4.5.0 +apache-airflow-providers-segment==3.4.0 +apache-airflow-providers-sendgrid==3.4.0 +apache-airflow-providers-sftp==4.9.0 +apache-airflow-providers-singularity==3.4.0 +apache-airflow-providers-slack==8.6.1 +apache-airflow-providers-smtp==1.5.0 +apache-airflow-providers-snowflake==5.3.1 +apache-airflow-providers-sqlite==3.7.1 +apache-airflow-providers-ssh==3.10.1 +apache-airflow-providers-tableau==4.4.2 +apache-airflow-providers-tabular==1.4.1 +apache-airflow-providers-telegram==4.3.1 +apache-airflow-providers-trino==5.6.2 +apache-airflow-providers-vertica==3.7.1 +apache-airflow-providers-weaviate==1.3.2 +apache-airflow-providers-yandex==3.9.0 +apache-airflow-providers-zendesk==4.6.0 +apache-beam==2.54.0 +apispec==6.4.0 +apprise==1.7.2 +argcomplete==3.2.2 asana==3.2.2 asgiref==3.7.2 asn1crypto==1.5.1 astroid==2.15.8 asttokens==2.4.1 async-timeout==4.0.3 +asyncssh==2.14.2 atlasclient==1.0.0 -atlassian-python-api==3.41.3 -attrs==23.1.0 -aws-sam-translator==1.79.0 +atlassian-python-api==3.41.10 +attrs==23.2.0 +aws-sam-translator==1.85.0 aws-xray-sdk==2.12.1 -azure-batch==14.0.0 +azure-batch==14.1.0 azure-common==1.1.28 -azure-core==1.29.5 +azure-core==1.30.0 azure-cosmos==4.5.1 azure-datalake-store==0.0.53 azure-identity==1.15.0 -azure-keyvault-secrets==4.7.0 -azure-kusto-data==4.2.0 +azure-keyvault-secrets==4.8.0 +azure-kusto-data==4.3.1 azure-mgmt-containerinstance==10.1.0 -azure-mgmt-containerregistry==10.2.0 +azure-mgmt-containerregistry==10.3.0 azure-mgmt-core==1.4.0 -azure-mgmt-cosmosdb==9.3.0 -azure-mgmt-datafactory==3.1.0 +azure-mgmt-cosmosdb==9.4.0 +azure-mgmt-datafactory==5.0.0 azure-mgmt-datalake-nspkg==3.0.1 azure-mgmt-datalake-store==0.5.0 azure-mgmt-nspkg==3.0.2 azure-mgmt-resource==23.0.1 azure-mgmt-storage==21.1.0 azure-nspkg==3.0.2 -azure-servicebus==7.11.3 -azure-storage-blob==12.18.3 -azure-storage-common==2.1.0 -azure-storage-file-datalake==12.13.2 -azure-storage-file-share==12.14.2 -azure-storage-file==2.1.0 +azure-servicebus==7.11.4 +azure-storage-blob==12.19.0 +azure-storage-file-datalake==12.14.0 +azure-storage-file-share==12.15.0 +azure-synapse-artifacts==0.18.0 azure-synapse-spark==0.7.0 -backoff==1.10.0 -bcrypt==4.0.1 -beautifulsoup4==4.12.2 -billiard==4.1.0 -bitarray==2.8.2 -black==23.10.1 -blinker==1.6.3 -boto3==1.28.64 -botocore==1.31.64 +backoff==2.2.1 +bcrypt==4.1.2 +beautifulsoup4==4.12.3 +billiard==4.2.0 +bitarray==2.9.2 +black==24.2.0 +blinker==1.7.0 +boto3==1.33.13 +botocore==1.33.13 cachelib==0.9.0 cachetools==5.3.2 -cassandra-driver==3.28.0 -cattrs==23.1.2 -celery==5.3.4 -certifi==2023.7.22 +cassandra-driver==3.29.0 +cattrs==23.2.3 +celery==5.3.6 +certifi==2024.2.2 cffi==1.16.0 cfgv==3.4.0 -cfn-lint==0.83.1 +cfn-lint==0.85.2 cgroupspy==0.2.2 chardet==5.2.0 charset-normalizer==3.3.2 @@ -245,358 +249,375 @@ click==8.1.7 clickclick==20.10.2 cloudant==2.15.0 cloudpickle==2.2.1 +cohere==4.50 colorama==0.4.6 colorlog==4.8.0 +comm==0.2.1 confluent-kafka==2.3.0 connexion==2.14.2 -coverage==7.3.2 +coverage==7.4.3 crcmod==1.7 -cron-descriptor==1.4.0 +cron-descriptor==1.4.3 croniter==2.0.1 -cryptography==41.0.5 +cryptography==41.0.7 curlify==2.2.1 -dask==2023.10.1 -databricks-sql-connector==2.9.3 -datadog==0.47.0 -db-dtypes==1.1.1 +databricks-sql-connector==2.9.4 +datadog==0.48.0 +db-dtypes==1.2.0 +debugpy==1.8.1 decorator==5.1.1 defusedxml==0.7.1 -deprecation==2.1.0 +deltalake==0.15.3 dill==0.3.1.1 -distlib==0.3.7 -distributed==2023.10.1 -dnspython==2.4.2 -docker==6.1.3 +distlib==0.3.8 +distro==1.9.0 +dnspython==2.6.1 +docker==7.0.0 docopt==0.6.2 docutils==0.20.1 -duckdb==0.9.1 -ecdsa==0.18.0 -elastic-transport==8.10.0 -elasticsearch==8.10.1 +duckdb==0.10.0 +editables==0.5 +elastic-transport==8.12.0 +elasticsearch==8.12.1 email-validator==1.3.1 entrypoints==0.4 +enum34==1.1.10 eralchemy2==1.3.8 et-xmlfile==1.1.0 -eventlet==0.33.3 -exceptiongroup==1.1.3 +eventlet==0.35.2 +exceptiongroup==1.2.0 execnet==2.0.2 executing==2.0.1 -facebook-business==18.0.3 -fastavro==1.9.0 +facebook_business==19.0.0 +fastavro==1.9.4 fasteners==0.19 -fastjsonschema==2.18.1 +fastjsonschema==2.19.1 filelock==3.13.1 flower==2.0.1 -frozenlist==1.4.0 -fsspec==2023.10.0 -future==0.18.3 +frozenlist==1.4.1 +fsspec==2024.2.0 +future==1.0.0 gcloud-aio-auth==4.2.3 -gcloud-aio-bigquery==7.0.0 -gcloud-aio-storage==9.0.0 -gcsfs==2023.10.0 +gcloud-aio-bigquery==7.1.0 +gcloud-aio-storage==9.2.0 +gcsfs==2024.2.0 geomet==0.2.1.post1 -gevent==23.9.1 +gevent==24.2.1 gitdb==4.0.11 -google-ads==22.1.0 -google-api-core==2.12.0 -google-api-python-client==2.106.0 -google-auth-httplib2==0.1.1 -google-auth-oauthlib==1.1.0 -google-auth==2.23.4 -google-cloud-aiplatform==1.36.0 -google-cloud-appengine-logging==1.3.2 +google-ads==23.1.0 +google-analytics-admin==0.22.6 +google-api-core==2.17.1 +google-api-python-client==2.119.0 +google-auth-httplib2==0.2.0 +google-auth-oauthlib==1.2.0 +google-auth==2.28.1 +google-cloud-aiplatform==1.42.1 +google-cloud-appengine-logging==1.4.2 google-cloud-audit-log==0.2.5 -google-cloud-automl==2.11.3 -google-cloud-batch==0.17.2 -google-cloud-bigquery-datatransfer==3.12.1 -google-cloud-bigquery-storage==2.22.0 -google-cloud-bigquery==3.13.0 -google-cloud-bigtable==2.21.0 -google-cloud-build==3.20.1 -google-cloud-compute==1.14.1 -google-cloud-container==2.32.0 -google-cloud-core==2.3.3 -google-cloud-datacatalog==3.16.0 -google-cloud-dataflow-client==0.8.5 -google-cloud-dataform==0.5.3 -google-cloud-dataplex==1.7.0 -google-cloud-dataproc-metastore==1.13.0 -google-cloud-dataproc==5.6.0 -google-cloud-dlp==3.12.3 -google-cloud-kms==2.19.2 -google-cloud-language==2.11.1 -google-cloud-logging==3.8.0 -google-cloud-memcache==1.7.3 -google-cloud-monitoring==2.16.0 -google-cloud-orchestration-airflow==1.9.2 -google-cloud-os-login==2.11.0 -google-cloud-pubsub==2.18.4 -google-cloud-redis==2.13.2 -google-cloud-resource-manager==1.10.4 -google-cloud-run==0.10.0 -google-cloud-secret-manager==2.16.4 -google-cloud-spanner==3.40.1 -google-cloud-speech==2.21.1 -google-cloud-storage-transfer==1.9.2 -google-cloud-storage==2.13.0 -google-cloud-tasks==2.14.2 -google-cloud-texttospeech==2.14.2 -google-cloud-translate==3.12.1 -google-cloud-videointelligence==2.11.4 -google-cloud-vision==3.4.5 -google-cloud-workflows==1.12.1 +google-cloud-automl==2.13.2 +google-cloud-batch==0.17.12 +google-cloud-bigquery-datatransfer==3.15.0 +google-cloud-bigquery-storage==2.24.0 +google-cloud-bigquery==3.17.2 +google-cloud-bigtable==2.23.0 +google-cloud-build==3.23.2 +google-cloud-compute==1.17.0 +google-cloud-container==2.41.0 +google-cloud-core==2.4.1 +google-cloud-datacatalog==3.18.2 +google-cloud-dataflow-client==0.8.9 +google-cloud-dataform==0.5.8 +google-cloud-dataplex==1.12.2 +google-cloud-dataproc-metastore==1.15.2 +google-cloud-dataproc==5.9.2 +google-cloud-dlp==3.15.2 +google-cloud-kms==2.21.2 +google-cloud-language==2.13.2 +google-cloud-logging==3.9.0 +google-cloud-memcache==1.9.2 +google-cloud-monitoring==2.19.2 +google-cloud-orchestration-airflow==1.12.0 +google-cloud-os-login==2.14.2 +google-cloud-pubsub==2.19.6 +google-cloud-redis==2.15.2 +google-cloud-resource-manager==1.12.2 +google-cloud-run==0.10.4 +google-cloud-secret-manager==2.18.2 +google-cloud-spanner==3.42.0 +google-cloud-speech==2.25.0 +google-cloud-storage-transfer==1.11.2 +google-cloud-storage==2.14.0 +google-cloud-tasks==2.16.2 +google-cloud-texttospeech==2.16.2 +google-cloud-translate==3.15.2 +google-cloud-videointelligence==2.13.2 +google-cloud-vision==3.7.1 +google-cloud-workflows==1.14.2 google-crc32c==1.5.0 google-re2==1.1 -google-resumable-media==2.6.0 -googleapis-common-protos==1.61.0 +google-resumable-media==2.7.0 +googleapis-common-protos==1.62.0 graphql-core==3.2.3 graphviz==0.20.1 -greenlet==3.0.1 -grpc-google-iam-v1==0.12.6 +greenlet==3.0.3 +grpc-google-iam-v1==0.13.0 +grpc-interceptor==0.15.4 grpcio-gcp==0.2.2 -grpcio-status==1.59.2 -grpcio==1.59.2 +grpcio-status==1.62.0 +grpcio==1.62.0 gssapi==1.8.3 gunicorn==21.2.0 h11==0.14.0 +hatch==1.9.3 +hatchling==1.21.1 hdfs==2.7.3 hmsclient==0.1.1 httpcore==0.16.3 httplib2==0.22.0 httpx==0.23.3 -humanize==4.8.0 -hvac==2.0.0 -identify==2.5.31 -idna==3.4 +humanize==4.9.0 +hvac==2.1.0 +hyperlink==21.0.0 +icdiff==2.0.7 +identify==2.5.35 +idna==3.6 ijson==3.2.3 imagesize==1.4.1 -importlib-metadata==6.8.0 -importlib-resources==6.1.0 -impyla==0.18.0 +importlib-metadata==6.11.0 +importlib-resources==6.1.1 +impyla==0.19.0 incremental==22.10.0 inflection==0.5.1 -influxdb-client==1.38.0 +influxdb-client==1.40.0 iniconfig==2.0.0 ipdb==0.13.13 -ipython==8.17.2 +ipykernel==6.29.2 +ipython==8.22.1 isodate==0.6.1 itsdangerous==2.1.2 -jaraco.classes==3.3.0 +jaraco.classes==3.3.1 jedi==0.19.1 jeepney==0.8.0 jmespath==0.10.0 +joserfc==0.9.0 jschema-to-python==1.2.3 json-merge-patch==0.2 jsondiff==2.0.0 jsonpatch==1.33 -jsonpath-ng==1.6.0 -jsonpickle==3.0.2 +jsonpath-ng==1.6.1 +jsonpickle==3.0.3 jsonpointer==2.4 -jsonschema-path==0.3.1 -jsonschema-specifications==2023.7.1 -jsonschema==4.19.2 +jsonschema-path==0.3.2 +jsonschema-specifications==2023.12.1 +jsonschema==4.21.1 junit-xml==1.9 -jupyter_client==8.5.0 -jupyter_core==5.5.0 -keyring==24.2.0 -kombu==5.3.2 +jupyter_client==8.6.0 +jupyter_core==5.7.1 +keyring==24.3.0 +kombu==5.3.5 krb5==0.5.1 -kubernetes-asyncio==24.2.3 -kubernetes==23.6.0 +kubernetes==29.0.0 +kubernetes_asyncio==29.0.0 kylinpy==2.8.4 -lazy-object-proxy==1.9.0 +lazy-object-proxy==1.10.0 ldap3==2.9.1 -limits==3.6.0 -linkify-it-py==2.0.2 -locket==1.0.0 +limits==3.9.0 +linkify-it-py==2.0.3 lockfile==0.12.2 -looker-sdk==23.16.0 -lxml==4.9.3 -lz4==4.3.2 +loguru==0.7.2 +looker-sdk==24.2.0 +lxml==5.1.0 +lz4==4.3.3 markdown-it-py==3.0.0 -marshmallow-oneofschema==3.0.1 +marshmallow-oneofschema==3.1.1 marshmallow-sqlalchemy==0.26.1 -marshmallow==3.20.1 +marshmallow==3.20.2 matplotlib-inline==0.1.6 mdit-py-plugins==0.4.0 mdurl==0.1.2 +mmhash3==3.0.1 mongomock==4.1.2 -monotonic==1.6 -more-itertools==10.1.0 -moto==4.2.7 +more-itertools==10.2.0 +moto==5.0.2 mpmath==1.3.0 -msal-extensions==1.0.0 -msal==1.24.1 -msgpack==1.0.7 +msal-extensions==1.1.0 +msal==1.27.0 msrest==0.7.1 msrestazure==0.6.4 multi-key-dict==2.0.3 -multidict==6.0.4 -mypy-boto3-appflow==1.28.42 -mypy-boto3-rds==1.28.76 -mypy-boto3-redshift-data==1.28.36 -mypy-boto3-s3==1.28.55 +multidict==6.0.5 +mypy-boto3-appflow==1.34.0 +mypy-boto3-rds==1.34.49 +mypy-boto3-redshift-data==1.34.0 +mypy-boto3-s3==1.34.14 mypy-extensions==1.0.0 -mypy==1.2.0 -mysql-connector-python==8.0.29 -mysqlclient==2.2.0 -nbclient==0.8.0 +mypy==1.8.0 +mysql-connector-python==8.3.0 +mysqlclient==2.2.4 +nbclient==0.9.0 nbformat==5.9.2 -neo4j==5.14.0 +neo4j==5.17.0 +nest-asyncio==1.6.0 networkx==3.2.1 -nh3==0.2.14 +nh3==0.2.15 nodeenv==1.8.0 numpy==1.24.4 oauthlib==3.2.2 -objsize==0.6.1 +objsize==0.7.0 +openai==1.12.0 openapi-schema-validator==0.6.2 openapi-spec-validator==0.7.1 -openlineage-integration-common==1.4.1 -openlineage-python==1.4.1 -openlineage_sql==1.4.1 +openlineage-integration-common==1.9.0 +openlineage-python==1.9.0 +openlineage_sql==1.9.0 openpyxl==3.1.2 -opensearch-py==2.3.2 -opentelemetry-api==1.20.0 -opentelemetry-exporter-otlp-proto-common==1.20.0 -opentelemetry-exporter-otlp-proto-grpc==1.20.0 -opentelemetry-exporter-otlp-proto-http==1.20.0 -opentelemetry-exporter-otlp==1.20.0 -opentelemetry-exporter-prometheus==1.12.0rc1 -opentelemetry-proto==1.20.0 -opentelemetry-sdk==1.20.0 -opentelemetry-semantic-conventions==0.41b0 +opensearch-py==2.4.2 +opentelemetry-api==1.23.0 +opentelemetry-exporter-otlp-proto-common==1.23.0 +opentelemetry-exporter-otlp-proto-grpc==1.23.0 +opentelemetry-exporter-otlp-proto-http==1.23.0 +opentelemetry-exporter-otlp==1.23.0 +opentelemetry-exporter-prometheus==0.44b0 +opentelemetry-proto==1.23.0 +opentelemetry-sdk==1.23.0 +opentelemetry-semantic-conventions==0.44b0 opsgenie-sdk==2.1.5 -oracledb==1.4.2 +oracledb==2.0.1 ordered-set==4.1.0 -orjson==3.9.10 -oscrypto==1.3.0 -oss2==2.18.3 +orjson==3.9.15 +oss2==2.18.4 packaging==23.2 -pandas-gbq==0.19.2 -pandas==2.1.2 -papermill==2.4.0 -paramiko==3.3.1 +pandas-gbq==0.21.0 +pandas-stubs==2.0.2.230605 +pandas==2.1.4 +papermill==2.5.0 +paramiko==3.4.0 parso==0.8.3 -partd==1.4.1 pathable==0.4.3 -pathspec==0.11.2 -pbr==5.11.1 -pdpyras==5.1.2 -pendulum==2.1.2 -pexpect==4.8.0 -pinotdb==5.1.0 -pipdeptree==2.13.0 -pipx==1.2.1 +pathspec==0.12.1 +pbr==6.0.0 +pdpyras==5.2.0 +pendulum==3.0.0 +pexpect==4.9.0 +pgvector==0.2.5 +pinecone-client==2.2.4 +pinotdb==5.1.3 +pipdeptree==2.15.1 +pipx==1.4.3 pkginfo==1.9.6 platformdirs==3.11.0 -pluggy==1.3.0 +pluggy==1.4.0 ply==3.11 -plyvel==1.5.0 +plyvel==1.5.1 portalocker==2.8.2 -pre-commit==3.5.0 +pprintpp==0.4.0 +pre-commit==3.6.2 presto-python-client==0.8.4 prison==0.2.1 -prometheus-client==0.18.0 -prompt-toolkit==3.0.39 -proto-plus==1.22.3 -protobuf==4.24.4 -psutil==5.9.6 +prometheus_client==0.20.0 +prompt-toolkit==3.0.43 +proto-plus==1.23.0 +protobuf==4.25.3 +psutil==5.9.8 psycopg2-binary==2.9.9 ptyprocess==0.7.0 pure-eval==0.2.2 pure-sasl==0.6.2 -py-partiql-parser==0.4.1 +py-partiql-parser==0.5.1 py4j==0.10.9.7 -pyOpenSSL==23.3.0 -pyarrow==11.0.0 +pyOpenSSL==24.0.0 +pyarrow-hotfix==0.6 +pyarrow==14.0.2 pyasn1-modules==0.3.0 -pyasn1==0.5.0 -pycountry==22.3.5 +pyasn1==0.5.1 +pycountry==23.12.11 pycparser==2.21 -pycryptodome==3.19.0 -pycryptodomex==3.19.0 -pydantic==2.4.2 -pydantic_core==2.10.1 +pycryptodome==3.20.0 +pydantic==2.6.2 +pydantic_core==2.16.3 pydata-google-auth==1.8.2 pydot==1.4.2 -pydruid==0.6.5 +pydruid==0.6.6 pyenchant==3.2.2 pyexasol==0.25.2 -pygraphviz==1.11 +pygraphviz==1.12 +pyiceberg==0.6.0 pyjsparser==2.7.1 pykerberos==1.2.4 -pymongo==4.5.0 -pymssql==2.2.10 -pyodbc==5.0.1 +pymongo==4.6.2 +pymssql==2.2.11 +pyodbc==5.1.0 pyparsing==3.1.1 pypsrp==0.8.1 pyspark==3.5.0 pyspnego==0.10.2 -pytest-asyncio==0.21.1 +pytest-asyncio==0.23.5 pytest-cov==4.1.0 -pytest-httpx==0.21.3 +pytest-icdiff==0.9 pytest-instafail==0.5.0 pytest-mock==3.12.0 -pytest-rerunfailures==12.0 +pytest-rerunfailures==13.0 pytest-timeouts==1.2.1 -pytest-xdist==3.3.1 -pytest==7.4.3 -python-arango==7.7.0 +pytest-xdist==3.5.0 +pytest==7.4.4 +python-arango==7.9.1 python-daemon==3.0.1 python-dateutil==2.8.2 -python-dotenv==1.0.0 +python-dotenv==1.0.1 python-http-client==3.3.7 -python-jenkins==1.7.0 -python-jose==3.3.0 -python-ldap==3.4.3 +python-jenkins==1.8.2 +python-ldap==3.4.4 python-nvd3==0.15.0 -python-slugify==8.0.1 +python-slugify==8.0.4 python-telegram-bot==20.2 -pytz==2023.3.post1 -pytzdata==2020.1 +python3-saml==1.16.0 +pytz==2024.1 pywinrm==0.4.3 -pyzmq==25.1.1 +pyzmq==25.1.2 reactivex==4.0.4 readme-renderer==42.0 redis==4.6.0 -redshift-connector==2.0.915 -referencing==0.30.2 -regex==2023.10.3 -requests-file==1.5.1 +redshift-connector==2.1.0 +referencing==0.33.0 +regex==2023.12.25 +requests-file==2.0.0 requests-kerberos==0.14.0 requests-mock==1.11.0 requests-ntlm==1.2.0 requests-oauthlib==1.3.1 requests-toolbelt==1.0.0 requests==2.31.0 -responses==0.23.3 +responses==0.25.0 +restructuredtext-lint==1.4.0 rfc3339-validator==0.1.4 rfc3986==1.5.0 rich-argparse==1.4.0 -rich-click==1.7.1 -rich==13.6.0 -rpds-py==0.10.6 +rich-click==1.7.3 +rich==13.7.0 +rpds-py==0.18.0 rsa==4.9 -ruff==0.1.3 -s3transfer==0.7.0 +ruff==0.2.1 +s3fs==2024.2.0 +s3transfer==0.8.2 sarif-om==1.0.4 scramp==1.4.4 scrapbook==0.5.0 semver==3.0.2 -sendgrid==6.10.0 +sendgrid==6.11.0 sentinels==1.0.0 -sentry-sdk==1.33.1 +sentry-sdk==1.40.5 setproctitle==1.3.3 -shapely==2.0.2 +shapely==2.0.3 +shellingham==1.5.4 simple-salesforce==1.12.5 six==1.16.0 -slack-sdk==3.23.0 -smbprotocol==1.11.0 +slack_sdk==3.27.0 +smbprotocol==1.12.0 smmap==5.0.1 sniffio==1.3.0 snowballstemmer==2.2.0 -snowflake-connector-python==3.3.1 -snowflake-sqlalchemy==1.5.0 +snowflake-connector-python==3.7.1 +snowflake-sqlalchemy==1.5.1 sortedcontainers==2.4.0 soupsieve==2.5 sphinx-airflow-theme==0.0.12 @@ -604,90 +625,94 @@ sphinx-argparse==0.4.0 sphinx-autoapi==2.1.1 sphinx-copybutton==0.5.2 sphinx-jinja==2.0.2 -sphinx-rtd-theme==1.3.0 -sphinxcontrib-applehelp==1.0.7 -sphinxcontrib-devhelp==1.0.5 -sphinxcontrib-htmlhelp==2.0.4 +sphinx-rtd-theme==2.0.0 +sphinx_design==0.5.0 +sphinxcontrib-applehelp==1.0.8 +sphinxcontrib-devhelp==1.0.6 +sphinxcontrib-htmlhelp==2.0.5 sphinxcontrib-httpdomain==1.8.1 sphinxcontrib-jquery==4.1 sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-qthelp==1.0.6 +sphinxcontrib-qthelp==1.0.7 sphinxcontrib-redoc==1.6.0 -sphinxcontrib-serializinghtml==1.1.9 +sphinxcontrib-serializinghtml==1.1.5 sphinxcontrib-spelling==8.0.0 -spython==0.3.1 -sqlalchemy-bigquery==1.8.0 +spython==0.3.13 +sqlalchemy-bigquery==1.9.0 sqlalchemy-drill==1.1.4 sqlalchemy-redshift==0.8.14 sqlalchemy-spanner==1.6.2 sqlparse==0.4.4 -sshpubkeys==3.3.1 sshtunnel==0.4.0 stack-data==0.6.3 starkbank-ecdsa==2.2.0 statsd==4.0.1 +strictyaml==1.7.3 sympy==1.12 -tableauserverclient==0.25 +tableauserverclient==0.30 tabulate==0.9.0 -tblib==3.0.0 tenacity==8.2.3 -termcolor==2.3.0 +termcolor==2.4.0 text-unidecode==1.3 -textwrap3==0.9.2 thrift-sasl==0.4.3 thrift==0.16.0 time-machine==2.13.0 tomli==2.0.1 -tomlkit==0.12.1 -toolz==0.12.0 -tornado==6.3.3 -towncrier==23.10.0 -tqdm==4.66.1 -traitlets==5.13.0 -trino==0.327.0 -twine==4.0.2 -types-Deprecated==1.2.9.3 -types-Markdown==3.5.0.0 +tomli_w==1.0.0 +tomlkit==0.12.3 +tornado==6.4 +towncrier==23.11.0 +tqdm==4.66.2 +traitlets==5.14.1 +trino==0.328.0 +trove-classifiers==2024.2.23 +twine==5.0.0 +types-Deprecated==1.2.9.20240106 +types-Markdown==3.5.0.20240129 types-PyMySQL==1.1.0.1 types-PyYAML==6.0.12.12 +types-aiofiles==23.2.0.20240106 types-certifi==2021.10.8.3 -types-croniter==2.0.0.0 -types-docutils==0.20.0.3 -types-paramiko==3.3.0.0 -types-protobuf==4.24.0.4 -types-pyOpenSSL==23.3.0.0 -types-python-dateutil==2.8.19.14 -types-python-slugify==8.0.0.3 -types-pytz==2023.3.1.1 -types-redis==4.6.0.8 -types-requests==2.31.0.6 -types-setuptools==68.2.0.0 -types-tabulate==0.9.0.3 +types-croniter==2.0.0.20240106 +types-docutils==0.20.0.20240201 +types-paramiko==3.4.0.20240205 +types-protobuf==4.24.0.20240129 +types-pyOpenSSL==24.0.0.20240130 +types-python-dateutil==2.8.19.20240106 +types-python-slugify==8.0.2.20240127 +types-pytz==2024.1.0.20240203 +types-redis==4.6.0.20240218 +types-requests==2.31.0.20240218 +types-setuptools==69.1.0.20240223 +types-tabulate==0.9.0.20240106 types-termcolor==1.1.6.2 types-toml==0.10.8.7 -types-urllib3==1.26.25.14 -typing_extensions==4.8.0 -tzdata==2023.3 +typing_extensions==4.9.0 +tzdata==2024.1 tzlocal==5.2 -uc-micro-py==1.0.2 +uc-micro-py==1.0.3 unicodecsv==0.14.1 +universal-pathlib==0.1.4 uritemplate==4.1.1 -urllib3==1.26.18 +urllib3==2.0.7 userpath==1.9.1 -vertica-python==1.3.6 -vine==5.0.0 -virtualenv==20.24.6 +validators==0.22.0 +vertica-python==1.3.8 +vine==5.1.0 +virtualenv==20.25.1 watchtower==3.0.1 -wcwidth==0.2.9 -websocket-client==1.6.4 -wrapt==1.15.0 +wcwidth==0.2.13 +weaviate-client==3.26.2 +websocket-client==1.7.0 +wrapt==1.16.0 +xmlsec==1.3.13 xmltodict==0.13.0 -yamllint==1.32.0 -yarl==1.9.2 +yamllint==1.35.1 +yandexcloud==0.260.0 +yarl==1.9.4 zeep==4.2.1 -zenpy==2.0.41 -zict==3.0.0 +zenpy==2.0.47 zipp==3.17.0 zope.event==5.0 -zope.interface==6.1 +zope.interface==6.2 zstandard==0.22.0 diff --git a/docker-compose.yml b/docker-compose.yml index 1a2a35c42..a91181c03 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,5 @@ # Adapted from official docker-compose.yaml -# https://airflow.apache.org/docs/apache-airflow/2.7.3/docker-compose.yaml +# https://airflow.apache.org/docs/apache-airflow/2.8.2/docker-compose.yaml # # WARNING: This configuration is for local development. Do not use it in a production deployment. # @@ -11,7 +11,6 @@ # AIRFLOW_PROJ_DIR - Base path to which all the files will be volumed. # Default: . --- -version: '3.8' x-airflow-common: &airflow-common build: . @@ -22,8 +21,6 @@ x-airflow-common: AIRFLOW__CELERY__FLOWER_PORT: '5555' AIRFLOW__CORE__EXECUTOR: CeleryExecutor AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow - # For backward compatibility, with Airflow <2.3 - AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://airflow:airflow@postgres/airflow AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0 AIRFLOW__CORE__FERNET_KEY: $FERNET_KEY @@ -159,20 +156,6 @@ services: command: - -c - | - function ver() { - printf "%04d%04d%04d%04d" $${1//./ } - } - airflow_version=$$(AIRFLOW__LOGGING__LOGGING_LEVEL=INFO && gosu airflow airflow version) - airflow_version_comparable=$$(ver $${airflow_version}) - min_airflow_version=2.2.0 - min_airflow_version_comparable=$$(ver $${min_airflow_version}) - if (( airflow_version_comparable < min_airflow_version_comparable )); then - echo - echo -e "\033[1;31mERROR!!!: Too old Airflow version $${airflow_version}!\e[0m" - echo "The minimum Airflow version supported: $${min_airflow_version}. Only use this or higher!" - echo - exit 1 - fi if [[ -z "${AIRFLOW_UID}" ]]; then echo echo -e "\033[1;33mWARNING!!!: AIRFLOW_UID not set!\e[0m" diff --git a/pyproject.toml b/pyproject.toml index e593760ac..7af88bc16 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,8 +1,10 @@ -[tool.ruff.isort] +[tool.ruff.lint.isort] known-third-party = ["airflow"] [tool.ruff] target-version = "py310" + +[tool.ruff.lint] select = [ "E", # pycodestyle "W", # pycodestyle diff --git a/requirements.in b/requirements.in index ae3da3f76..49528c085 100644 --- a/requirements.in +++ b/requirements.in @@ -1,10 +1,10 @@ # Official Airflow constraints file # Doc: https://airflow.apache.org/docs/apache-airflow/stable/installation/installing-from-pypi.html#constraints-files -# File: https://raw.githubusercontent.com/apache/airflow/constraints-2.5.3/constraints-3.10.txt +# File: https://raw.githubusercontent.com/apache/airflow/constraints-2.8.2/constraints-3.10.txt --constraint ./constraints.txt # Airflow dependencies -apache-airflow[async,password,statsd]==2.7.3 +apache-airflow[async,google-auth,password,statsd]==2.8.2 apache-airflow-providers-amazon apache-airflow-providers-celery apache-airflow-providers-cncf-kubernetes @@ -16,10 +16,10 @@ apache-airflow-providers-slack airflow-provider-fivetran-async==2.0.2 # Code quality -pytest==7.4.3 +pytest==7.4.4 pytest-mock==3.12.0 -black==23.10.1 -ruff==0.1.3 +black==24.2.0 +ruff==0.2.1 # Required for /app/dags/fivetran_acoustic.py, /app/dags/utils/acoustic/acoustic_client.py xmltodict diff --git a/requirements.txt b/requirements.txt index 65b068f46..6b00d6303 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,46 +5,47 @@ # pip-compile --no-annotate --strip-extras # aiofiles==23.2.1 -aiohttp==3.8.6 +aiohttp==3.9.3 aiosignal==1.3.1 airflow-provider-fivetran-async==2.0.2 -alembic==1.12.1 -amqp==5.1.1 -annotated-types==0.6.0 -anyio==4.0.0 -apache-airflow==2.7.3 -apache-airflow-providers-amazon==8.10.0 -apache-airflow-providers-celery==3.4.1 -apache-airflow-providers-cncf-kubernetes==7.8.0 -apache-airflow-providers-common-sql==1.8.0 -apache-airflow-providers-ftp==3.6.0 -apache-airflow-providers-google==10.11.0 -apache-airflow-providers-http==4.6.0 -apache-airflow-providers-imap==3.4.0 -apache-airflow-providers-postgres==5.7.1 -apache-airflow-providers-redis==3.4.0 -apache-airflow-providers-slack==8.3.0 -apache-airflow-providers-sqlite==3.5.0 -apispec==6.3.0 -argcomplete==3.1.3 +alembic==1.13.1 +amqp==5.2.0 +anyio==4.3.0 +apache-airflow==2.8.2 +apache-airflow-providers-amazon==8.18.0 +apache-airflow-providers-celery==3.6.0 +apache-airflow-providers-cncf-kubernetes==8.0.0 +apache-airflow-providers-common-io==1.3.0 +apache-airflow-providers-common-sql==1.11.0 +apache-airflow-providers-ftp==3.7.0 +apache-airflow-providers-google==10.15.0 +apache-airflow-providers-http==4.9.1 +apache-airflow-providers-imap==3.5.0 +apache-airflow-providers-postgres==5.10.1 +apache-airflow-providers-redis==3.6.0 +apache-airflow-providers-slack==8.6.1 +apache-airflow-providers-sqlite==3.7.1 +apispec==6.4.0 +argcomplete==3.2.2 asgiref==3.7.2 asn1crypto==1.5.1 async-timeout==4.0.3 -attrs==23.1.0 -babel==2.13.1 -backoff==1.10.0 -bcrypt==4.0.1 -beautifulsoup4==4.12.2 -billiard==4.1.0 -black==23.10.1 -blinker==1.6.3 -boto3==1.28.64 -botocore==1.31.64 +attrs==23.2.0 +authlib==1.3.0 +babel==2.14.0 +backoff==2.2.1 +bcrypt==4.1.2 +beautifulsoup4==4.12.3 +billiard==4.2.0 +black==24.2.0 +blinker==1.7.0 +boto3==1.33.13 +botocore==1.33.13 cachelib==0.9.0 cachetools==5.3.2 -cattrs==23.1.2 -celery==5.3.4 -certifi==2023.7.22 +cattrs==23.2.3 +celery==5.3.6 +certifi==2024.2.2 cffi==1.16.0 chardet==5.2.0 charset-normalizer==3.3.2 @@ -55,205 +56,204 @@ click-repl==0.3.0 clickclick==20.10.2 colorama==0.4.6 colorlog==4.8.0 -configupdater==3.1.1 +configupdater==3.2 connexion==2.14.2 -cron-descriptor==1.4.0 +cron-descriptor==1.4.3 croniter==2.0.1 -cryptography==41.0.5 -db-dtypes==1.1.1 +cryptography==41.0.7 +db-dtypes==1.2.0 decorator==5.1.1 deprecated==1.2.14 dill==0.3.1.1 -dnspython==2.4.2 +dnspython==2.6.1 docutils==0.20.1 email-validator==1.3.1 -eventlet==0.33.3 -exceptiongroup==1.1.3 +eventlet==0.35.2 +exceptiongroup==1.2.0 flask==2.2.5 -flask-appbuilder==4.3.6 +flask-appbuilder==4.3.11 flask-babel==2.0.0 flask-bcrypt==1.0.1 flask-caching==2.1.0 -flask-jwt-extended==4.5.3 -flask-limiter==3.5.0 +flask-jwt-extended==4.6.0 +flask-limiter==3.5.1 flask-login==0.6.3 flask-session==0.5.0 flask-sqlalchemy==2.5.1 flask-wtf==1.2.1 flower==2.0.1 -frozenlist==1.4.0 -fsspec==2023.10.0 +frozenlist==1.4.1 +fsspec==2024.2.0 gcloud-aio-auth==4.2.3 -gcloud-aio-bigquery==7.0.0 -gcloud-aio-storage==9.0.0 -gcsfs==2023.10.0 -gevent==23.9.1 -google-ads==22.1.0 -google-api-core==2.12.0 -google-api-python-client==2.106.0 -google-auth==2.23.4 -google-auth-httplib2==0.1.1 -google-auth-oauthlib==1.1.0 -google-cloud-aiplatform==1.36.0 -google-cloud-appengine-logging==1.3.2 +gcloud-aio-bigquery==7.1.0 +gcloud-aio-storage==9.2.0 +gcsfs==2024.2.0 +gevent==24.2.1 +google-ads==23.1.0 +google-analytics-admin==0.22.6 +google-api-core==2.17.1 +google-api-python-client==2.119.0 +google-auth==2.28.1 +google-auth-httplib2==0.2.0 +google-auth-oauthlib==1.2.0 +google-cloud-aiplatform==1.42.1 +google-cloud-appengine-logging==1.4.2 google-cloud-audit-log==0.2.5 -google-cloud-automl==2.11.3 -google-cloud-batch==0.17.2 -google-cloud-bigquery==3.13.0 -google-cloud-bigquery-datatransfer==3.12.1 -google-cloud-bigquery-storage==2.22.0 -google-cloud-bigtable==2.21.0 -google-cloud-build==3.20.1 -google-cloud-compute==1.14.1 -google-cloud-container==2.32.0 -google-cloud-core==2.3.3 -google-cloud-datacatalog==3.16.0 -google-cloud-dataflow-client==0.8.5 -google-cloud-dataform==0.5.3 -google-cloud-dataplex==1.7.0 -google-cloud-dataproc==5.6.0 -google-cloud-dataproc-metastore==1.13.0 -google-cloud-dlp==3.12.3 -google-cloud-kms==2.19.2 -google-cloud-language==2.11.1 -google-cloud-logging==3.8.0 -google-cloud-memcache==1.7.3 -google-cloud-monitoring==2.16.0 -google-cloud-orchestration-airflow==1.9.2 -google-cloud-os-login==2.11.0 -google-cloud-pubsub==2.18.4 -google-cloud-redis==2.13.2 -google-cloud-resource-manager==1.10.4 -google-cloud-run==0.10.0 -google-cloud-secret-manager==2.16.4 -google-cloud-spanner==3.40.1 -google-cloud-speech==2.21.1 -google-cloud-storage==2.13.0 -google-cloud-storage-transfer==1.9.2 -google-cloud-tasks==2.14.2 -google-cloud-texttospeech==2.14.2 -google-cloud-translate==3.12.1 -google-cloud-videointelligence==2.11.4 -google-cloud-vision==3.4.5 -google-cloud-workflows==1.12.1 +google-cloud-automl==2.13.2 +google-cloud-batch==0.17.12 +google-cloud-bigquery==3.17.2 +google-cloud-bigquery-datatransfer==3.15.0 +google-cloud-bigquery-storage==2.24.0 +google-cloud-bigtable==2.23.0 +google-cloud-build==3.23.2 +google-cloud-compute==1.17.0 +google-cloud-container==2.41.0 +google-cloud-core==2.4.1 +google-cloud-datacatalog==3.18.2 +google-cloud-dataflow-client==0.8.9 +google-cloud-dataform==0.5.8 +google-cloud-dataplex==1.12.2 +google-cloud-dataproc==5.9.2 +google-cloud-dataproc-metastore==1.15.2 +google-cloud-dlp==3.15.2 +google-cloud-kms==2.21.2 +google-cloud-language==2.13.2 +google-cloud-logging==3.9.0 +google-cloud-memcache==1.9.2 +google-cloud-monitoring==2.19.2 +google-cloud-orchestration-airflow==1.12.0 +google-cloud-os-login==2.14.2 +google-cloud-pubsub==2.19.6 +google-cloud-redis==2.15.2 +google-cloud-resource-manager==1.12.2 +google-cloud-run==0.10.4 +google-cloud-secret-manager==2.18.2 +google-cloud-spanner==3.42.0 +google-cloud-speech==2.25.0 +google-cloud-storage==2.14.0 +google-cloud-storage-transfer==1.11.2 +google-cloud-tasks==2.16.2 +google-cloud-texttospeech==2.16.2 +google-cloud-translate==3.15.2 +google-cloud-videointelligence==2.13.2 +google-cloud-vision==3.7.1 +google-cloud-workflows==1.14.2 google-crc32c==1.5.0 google-re2==1.1 -google-resumable-media==2.6.0 -googleapis-common-protos==1.61.0 -graphviz==0.20.1 -greenlet==3.0.1 -grpc-google-iam-v1==0.12.6 -grpcio==1.59.2 +google-resumable-media==2.7.0 +googleapis-common-protos==1.62.0 +greenlet==3.0.3 +grpc-google-iam-v1==0.13.0 +grpc-interceptor==0.15.4 +grpcio==1.62.0 grpcio-gcp==0.2.2 -grpcio-status==1.59.2 +grpcio-status==1.62.0 gunicorn==21.2.0 h11==0.14.0 httpcore==0.16.3 httplib2==0.22.0 httpx==0.23.3 -humanize==4.8.0 -idna==3.4 -importlib-metadata==6.8.0 -importlib-resources==6.1.0 +humanize==4.9.0 +idna==3.6 +importlib-metadata==6.11.0 +importlib-resources==6.1.1 inflection==0.5.1 iniconfig==2.0.0 itsdangerous==2.1.2 -jinja2==3.1.2 +jinja2==3.1.3 jmespath==0.10.0 json-merge-patch==0.2 -jsonpath-ng==1.6.0 -jsonschema==4.19.2 -jsonschema-specifications==2023.7.1 -kombu==5.3.2 -kubernetes==23.6.0 -kubernetes-asyncio==24.2.3 -lazy-object-proxy==1.9.0 -limits==3.6.0 -linkify-it-py==2.0.2 +jsonpath-ng==1.6.1 +jsonschema==4.21.1 +jsonschema-specifications==2023.12.1 +kombu==5.3.5 +kubernetes==29.0.0 +kubernetes-asyncio==29.0.0 +lazy-object-proxy==1.10.0 +limits==3.9.0 +linkify-it-py==2.0.3 lockfile==0.12.2 -looker-sdk==23.16.0 -lxml==4.9.3 -mako==1.2.4 -markdown==3.5.1 +looker-sdk==24.2.0 +lxml==5.1.0 +mako==1.3.2 markdown-it-py==3.0.0 -markupsafe==2.1.3 -marshmallow==3.20.1 -marshmallow-oneofschema==3.0.1 +markupsafe==2.1.5 +marshmallow==3.20.2 +marshmallow-oneofschema==3.1.1 marshmallow-sqlalchemy==0.26.1 mdit-py-plugins==0.4.0 mdurl==0.1.2 -multidict==6.0.4 +more-itertools==10.2.0 +multidict==6.0.5 mypy-extensions==1.0.0 numpy==1.24.4 oauthlib==3.2.2 -opentelemetry-api==1.20.0 -opentelemetry-exporter-otlp==1.20.0 -opentelemetry-exporter-otlp-proto-common==1.20.0 -opentelemetry-exporter-otlp-proto-grpc==1.20.0 -opentelemetry-exporter-otlp-proto-http==1.20.0 -opentelemetry-proto==1.20.0 -opentelemetry-sdk==1.20.0 -opentelemetry-semantic-conventions==0.41b0 +opentelemetry-api==1.23.0 +opentelemetry-exporter-otlp==1.23.0 +opentelemetry-exporter-otlp-proto-common==1.23.0 +opentelemetry-exporter-otlp-proto-grpc==1.23.0 +opentelemetry-exporter-otlp-proto-http==1.23.0 +opentelemetry-proto==1.23.0 +opentelemetry-sdk==1.23.0 +opentelemetry-semantic-conventions==0.44b0 ordered-set==4.1.0 packaging==23.2 -pandas==2.1.2 -pandas-gbq==0.19.2 -pathspec==0.11.2 -pendulum==2.1.2 +pandas==2.1.4 +pandas-gbq==0.21.0 +pathspec==0.12.1 +pendulum==3.0.0 platformdirs==3.11.0 -pluggy==1.3.0 +pluggy==1.4.0 ply==3.11 prison==0.2.1 -prometheus-client==0.18.0 -prompt-toolkit==3.0.39 -proto-plus==1.22.3 -protobuf==4.24.4 -psutil==5.9.6 +prometheus-client==0.20.0 +prompt-toolkit==3.0.43 +proto-plus==1.23.0 +protobuf==4.25.3 +psutil==5.9.8 psycopg2-binary==2.9.9 -pyarrow==11.0.0 -pyasn1==0.5.0 +pyarrow==14.0.2 +pyasn1==0.5.1 pyasn1-modules==0.3.0 +pyathena==3.3.0 pycparser==2.21 -pydantic==2.4.2 -pydantic-core==2.10.1 pydata-google-auth==1.8.2 -pygments==2.16.1 +pygments==2.17.2 pyjwt==2.8.0 -pyopenssl==23.3.0 +pyopenssl==24.0.0 pyparsing==3.1.1 -pytest==7.4.3 +pytest==7.4.4 pytest-mock==3.12.0 python-daemon==3.0.1 python-dateutil==2.8.2 python-nvd3==0.15.0 -python-slugify==8.0.1 -pytz==2023.3.post1 -pytzdata==2020.1 +python-slugify==8.0.4 +pytz==2024.1 pyyaml==6.0.1 redis==4.6.0 -redshift-connector==2.0.915 -referencing==0.30.2 +redshift-connector==2.1.0 +referencing==0.33.0 requests==2.31.0 requests-oauthlib==1.3.1 requests-toolbelt==1.0.0 rfc3339-validator==0.1.4 rfc3986==1.5.0 -rich==13.6.0 +rich==13.7.0 rich-argparse==1.4.0 -rpds-py==0.10.6 +rpds-py==0.18.0 rsa==4.9 -ruff==0.1.3 -s3transfer==0.7.0 +ruff==0.2.1 +s3transfer==0.8.2 scramp==1.4.4 setproctitle==1.3.3 -shapely==2.0.2 +shapely==2.0.3 six==1.16.0 -slack-sdk==3.23.0 +slack-sdk==3.27.0 sniffio==1.3.0 soupsieve==2.5 -sqlalchemy==1.4.50 -sqlalchemy-bigquery==1.8.0 -sqlalchemy-jsonfield==1.0.1.post0 +sqlalchemy==1.4.51 +sqlalchemy-bigquery==1.9.0 +sqlalchemy-jsonfield==1.0.2 sqlalchemy-redshift==0.8.14 sqlalchemy-spanner==1.6.2 sqlalchemy-utils==0.41.1 @@ -261,28 +261,30 @@ sqlparse==0.4.4 statsd==4.0.1 tabulate==0.9.0 tenacity==8.2.3 -termcolor==2.3.0 +termcolor==2.4.0 text-unidecode==1.3 +time-machine==2.13.0 tomli==2.0.1 -tornado==6.3.3 -typing-extensions==4.8.0 -tzdata==2023.3 -uc-micro-py==1.0.2 +tornado==6.4 +typing-extensions==4.9.0 +tzdata==2024.1 +uc-micro-py==1.0.3 unicodecsv==0.14.1 +universal-pathlib==0.1.4 uritemplate==4.1.1 -urllib3==1.26.18 -vine==5.0.0 +urllib3==2.0.7 +vine==5.1.0 watchtower==3.0.1 -wcwidth==0.2.9 -websocket-client==1.6.4 +wcwidth==0.2.13 +websocket-client==1.7.0 werkzeug==2.2.3 -wrapt==1.15.0 -wtforms==3.0.1 +wrapt==1.16.0 +wtforms==3.1.2 xmltodict==0.13.0 -yarl==1.9.2 +yarl==1.9.4 zipp==3.17.0 zope-event==5.0 -zope-interface==6.1 +zope-interface==6.2 # The following packages are considered to be unsafe in a requirements file: # setuptools