Upgrade Airflow to 1.10.15 bridge release prior to 2.0

mozilla · Jul 20, 2021 · 9bd76d2 · 9bd76d2
1 parent c856247
commit 9bd76d2
Show file tree

Hide file tree

Showing 5 changed files with 158 additions and 118 deletions.
diff --git a/bin/run b/bin/run
@@ -156,11 +156,11 @@ init_variables() {
 
 case $1 in
   flower)
-    exec airflow flower
+    exec airflow celery flower
     ;;
   web)
-    airflow initdb
-    airflow upgradedb
+    airflow db init
+    airflow db upgrade
 
     # Only init connections in dev
     [ ! -z ${DEVELOPMENT+check} ] && init_connections && init_variables
@@ -169,7 +169,7 @@ case $1 in
     exec airflow webserver -p ${PORT} --workers 4
     ;;
   worker)
-    exec airflow worker
+    exec airflow celery worker
     ;;
   scheduler)
     exec airflow scheduler

diff --git a/bin/test-parse b/bin/test-parse
@@ -44,7 +44,7 @@ function get_errors_in_listing {
     # Parse the logs for ERROR messages, these typically correspond to python
     # exceptions in the DAG. In general, there should NOT be any errors when
     # runnning the local environment.
-    docker-compose exec web airflow list_dags | grep "ERROR"
+    docker-compose exec web airflow dags list | grep "ERROR"
 }
 
 

diff --git a/plugins/backfill/main.py b/plugins/backfill/main.py
@@ -162,9 +162,9 @@ def background(self):
         # Prepare the command and execute in background
         background_cmd = "screen -dmS {} ".format(screen_id)
         if clear == 'true':
-            background_cmd = background_cmd + 'airflow clear -c '
+            background_cmd = background_cmd + 'airflow tasks clear -c '
         elif clear == 'false':
-            background_cmd = background_cmd + 'airflow backfill '
+            background_cmd = background_cmd + 'airflow dags backfill '
 
         if use_task_regex == 'true':
             background_cmd = background_cmd + "-t {} ".format(task_regex)

diff --git a/requirements.in b/requirements.in
@@ -1,7 +1,16 @@
-boto3
-kombu==4.6.3 # CeleryExecutor issues with 1.10.2 supposedly fixed in 1.10.5 airflow, but still observed issues on 1.10.7
+boto3==1.15.18
+botocore<1.19.0,>=1.18.0
+kombu==4.6.10 # CeleryExecutor issues with 1.10.2 supposedly fixed in 1.10.5 airflow, but still observed issues on 1.10.7
+importlib-metadata==2.1.0
+argcomplete==1.12.2
+pandas-gbq==0.14.1
 # removed hdfs
-apache-airflow[celery,postgres,hive,jdbc,async,password,crypto,github_enterprise,datadog,statsd,s3,mysql,google_auth,gcp_api,kubernetes]==1.10.12
+apache-airflow[celery,postgres,hive,jdbc,async,password,crypto,github_enterprise,datadog,statsd,s3,mysql,google_auth,gcp_api,kubernetes]==1.10.15
+apache-airflow-upgrade-check
+# Airflow 2.0 backported providers
+apache-airflow-backport-providers-google
+apache-airflow-backport-providers-amazon
+apache-airflow-backport-providers-http
 cryptography>=3.2
 mozlogging
 retrying