Move operators to project root

mozilla · Sep 18, 2023 · 1e2b42e · 1e2b42e
1 parent 4798c75
commit 1e2b42e
Show file tree

Hide file tree

Showing 24 changed files with 22 additions and 21 deletions.
diff --git a/dags/adm_export.py b/dags/adm_export.py
@@ -5,7 +5,7 @@
 from airflow.sensors.external_task import ExternalTaskSensor
 
 from utils.constants import ALLOWED_STATES, FAILED_STATES
-from utils.operators.gcp_container_operator import GKEPodOperator
+from operators.gcp_container_operator import GKEPodOperator
 from utils.tags import Tag
 
 DOCS = """\

diff --git a/dags/bhr_collection.py b/dags/bhr_collection.py
@@ -24,7 +24,7 @@
 
 from utils.constants import ALLOWED_STATES, FAILED_STATES
 from utils.dataproc import get_dataproc_parameters, moz_dataproc_pyspark_runner
-from utils.operators.gcp_container_operator import GKEPodOperator
+from operators.gcp_container_operator import GKEPodOperator
 from utils.tags import Tag
 
 default_args = {

diff --git a/dags/burnham.py b/dags/burnham.py
@@ -13,8 +13,8 @@
 from airflow import DAG
 from airflow.operators.python import PythonOperator
 
-from utils.operators.bq_sensor import BigQuerySQLSensorOperator
-from utils.operators.gcp_container_operator import GKEPodOperator
+from operators.bq_sensor import BigQuerySQLSensorOperator
+from operators.gcp_container_operator import GKEPodOperator
 from utils.tags import Tag
 
 DOCS = """\

diff --git a/dags/clean_gke_pods.py b/dags/clean_gke_pods.py
@@ -2,7 +2,7 @@
 
 from airflow import DAG
 
-from utils.operators.gcp_container_operator import GKEPodOperator
+from operators.gcp_container_operator import GKEPodOperator
 from utils.tags import Tag
 
 docs = """

diff --git a/dags/crash_symbolication.py b/dags/crash_symbolication.py
@@ -14,7 +14,7 @@
 
 from utils.constants import ALLOWED_STATES, FAILED_STATES
 from utils.dataproc import get_dataproc_parameters, moz_dataproc_pyspark_runner
-from utils.operators.gcp_container_operator import GKEPodOperator
+from operators.gcp_container_operator import GKEPodOperator
 from utils.tags import Tag
 
 default_args = {

diff --git a/dags/data_monitoring.py b/dags/data_monitoring.py
@@ -2,7 +2,7 @@
 
 from airflow import DAG
 
-from utils.operators.gcp_container_operator import GKEPodOperator
+from operators.gcp_container_operator import GKEPodOperator
 from utils.tags import Tag
 
 DOCS = """\

diff --git a/dags/dim_active_users_aggregates.py b/dags/dim_active_users_aggregates.py
@@ -17,7 +17,7 @@
 from airflow.sensors.external_task import ExternalTaskSensor
 
 from utils.constants import ALLOWED_STATES, FAILED_STATES
-from utils.operators.gcp_container_operator import GKEPodOperator
+from operators.gcp_container_operator import GKEPodOperator
 from utils.tags import Tag
 
 default_args = {

diff --git a/dags/experiment_auto_sizing.py b/dags/experiment_auto_sizing.py
@@ -12,7 +12,7 @@
 from airflow.sensors.external_task import ExternalTaskSensor
 
 from utils.constants import ALLOWED_STATES, FAILED_STATES
-from utils.operators.gcp_container_operator import GKEPodOperator
+from operators.gcp_container_operator import GKEPodOperator
 from utils.tags import Tag
 
 default_args = {

diff --git a/dags/firefox_public_data_report.py b/dags/firefox_public_data_report.py
@@ -17,7 +17,7 @@
     moz_dataproc_pyspark_runner,
 )
 from utils.gcp import bigquery_etl_query
-from utils.operators.gcp_container_operator import GKEPodOperator
+from operators.gcp_container_operator import GKEPodOperator
 from utils.tags import Tag
 
 default_args = {

diff --git a/dags/glam_glean_imports.py b/dags/glam_glean_imports.py
@@ -8,7 +8,7 @@
 from airflow.utils.task_group import TaskGroup
 
 from utils.constants import ALLOWED_STATES, FAILED_STATES
-from utils.operators.gcp_container_operator import GKENatPodOperator
+from operators.gcp_container_operator import GKENatPodOperator
 from utils.tags import Tag
 
 default_args = {

diff --git a/dags/jetstream.py b/dags/jetstream.py
@@ -18,7 +18,7 @@
 from airflow.sensors.external_task import ExternalTaskSensor
 
 from utils.constants import ALLOWED_STATES, FAILED_STATES
-from utils.operators.gcp_container_operator import GKEPodOperator
+from operators.gcp_container_operator import GKEPodOperator
 from utils.tags import Tag
 
 default_args = {

diff --git a/dags/merino_jobs.py b/dags/merino_jobs.py
@@ -5,7 +5,7 @@
 from airflow.hooks.base import BaseHook
 from airflow.operators.email import EmailOperator
 
-from utils.operators.gcp_container_operator import GKEPodOperator
+from operators.gcp_container_operator import GKEPodOperator
 from utils.tags import Tag
 
 DOCS = """\

diff --git a/dags/operational_monitoring.py b/dags/operational_monitoring.py
@@ -3,7 +3,7 @@
 from airflow import DAG
 from airflow.sensors.external_task import ExternalTaskSensor
 
-from utils.operators.gcp_container_operator import GKEPodOperator
+from operators.gcp_container_operator import GKEPodOperator
 from utils.tags import Tag
 
 docs = """

diff --git a/dags/overwatch.py b/dags/overwatch.py
@@ -14,7 +14,7 @@
 
 from airflow import DAG
 
-from utils.operators.gcp_container_operator import GKEPodOperator
+from operators.gcp_container_operator import GKEPodOperator
 
 default_args = {
     "owner": "[email protected]",

diff --git a/dags/partybal.py b/dags/partybal.py
@@ -17,7 +17,7 @@
 
 from airflow import DAG
 
-from utils.operators.gcp_container_operator import GKEPodOperator
+from operators.gcp_container_operator import GKEPodOperator
 from utils.tags import Tag
 
 default_args = {

diff --git a/dags/probe_scraper.py b/dags/probe_scraper.py
@@ -11,7 +11,7 @@
 from airflow.providers.http.operators.http import SimpleHttpOperator
 from airflow.utils.weekday import WeekDay
 
-from utils.operators.gcp_container_operator import GKEPodOperator
+from operators.gcp_container_operator import GKEPodOperator
 from utils.tags import Tag
 
 DOCS = """\

diff --git a/dags/socorro_import.py b/dags/socorro_import.py
@@ -9,7 +9,7 @@
 from airflow.utils.task_group import TaskGroup
 
 from utils.dataproc import moz_dataproc_pyspark_runner
-from utils.operators.gcp_container_operator import GKEPodOperator
+from operators.gcp_container_operator import GKEPodOperator
 from utils.tags import Tag
 
 """

diff --git a/dags/taar_daily.py b/dags/taar_daily.py
@@ -26,7 +26,7 @@
     moz_dataproc_jar_runner,
     moz_dataproc_pyspark_runner,
 )
-from utils.operators.gcp_container_operator import GKEPodOperator
+from operators.gcp_container_operator import GKEPodOperator
 from utils.tags import Tag
 
 TAAR_ETL_STORAGE_BUCKET = Variable.get("taar_etl_storage_bucket")

diff --git a/dags/taar_weekly.py b/dags/taar_weekly.py
@@ -17,7 +17,7 @@
 from airflow.operators.subdag import SubDagOperator
 
 from utils.dataproc import moz_dataproc_pyspark_runner
-from utils.operators.gcp_container_operator import GKEPodOperator
+from operators.gcp_container_operator import GKEPodOperator
 from utils.tags import Tag
 
 taar_ensemble_cluster_name = "dataproc-taar-ensemble"

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -42,6 +42,7 @@ x-airflow-common:
     - ./logs:/opt/airflow/logs
     - ./plugins:/opt/airflow/plugins
     - ./utils:/opt/airflow/utils
+    - ./operators:/opt/airflow/operators
     - ./resources/dev_webserver_config.py:/opt/airflow/webserver_config.py
     - ./resources/dev_connections.json:/opt/airflow/dev_connections.json
     - ./resources/dev_variables.json:/opt/airflow/dev_variables.json

diff --git a/utils/operators/__init__.py → operators/__init__.py b/utils/operators/__init__.py → operators/__init__.py
diff --git a/utils/operators/bq_sensor.py → operators/bq_sensor.py b/utils/operators/bq_sensor.py → operators/bq_sensor.py
diff --git a/utils/operators/gcp_container_operator.py → operators/gcp_container_operator.py b/utils/operators/gcp_container_operator.py → operators/gcp_container_operator.py
diff --git a/utils/gcp.py b/utils/gcp.py
@@ -15,7 +15,7 @@
 )
 
 from utils.dataproc import get_dataproc_parameters
-from utils.operators.gcp_container_operator import GKEPodOperator
+from operators.gcp_container_operator import GKEPodOperator
 
 GCP_PROJECT_ID = "moz-fx-data-airflow-gke-prod"
 DATAPROC_PROJECT_ID = "airflow-dataproc"