try catch when loading dag

astronomer · Oct 25, 2023 · 9d8035a · 9d8035a
1 parent aad7600
commit 9d8035a
Showing 1 changed file with 9 additions and 5 deletions.
diff --git a/dagfactory/dagfactory.py b/dagfactory/dagfactory.py
@@ -189,11 +189,15 @@ def load_yaml_dags(
         suffix = [".yaml", ".yml"]
     candidate_dag_files = []
     for suf in suffix:
-        candidate_dag_files = chain(
+        candidate_dag_files = list(chain(
             candidate_dag_files, Path(dags_folder).rglob(f"*{suf}")
-        )
-
+        ))
+    logging.info(candidate_dag_files)
     for config_file_path in candidate_dag_files:
         config_file_abs_path = str(config_file_path.absolute())
-        DagFactory(config_file_abs_path).generate_dags(globals_dict)
-        logging.info("DAG loaded: %s", config_file_path)
+        logging.info(f"Loading {config_file_abs_path}")
+        try:
+            DagFactory(config_file_abs_path).generate_dags(globals_dict)
+            logging.info("DAG loaded: %s", config_file_path)
+        except Exception as e:
+            logging.warning(f"Failed to load dag from {config_file_path}: {e}")