openlineage: add method to common.compat to not force hooks to try/ex…

…cept every 2.10 hook lineage call (apache#40812)

Signed-off-by: Maciej Obuchowski <[email protected]>

.pre-commit-config.yaml

@@ -675,6 +675,15 @@ repos:
           (?x)
           ^airflow/providers/.*\.py$
         exclude: ^.*/.*_vendor/
+      - id: check-get-lineage-collector-providers
+        language: pygrep
+        name: Check providers import hook lineage code from compat
+        description: Make sure you import from airflow.provider.common.compat.lineage.hook instead of
+          airflow.lineage.hook.
+        entry: "airflow\\.lineage\\.hook"
+        pass_filenames: true
+        files: ^airflow/providers/.*\.py$
+        exclude: ^airflow/providers/common/compat/.*\.py$
       - id: check-decorated-operator-implements-custom-name
         name: Check @task decorator implements custom_operator_name
         language: python

tests/providers/common/compat/test_empty.py → airflow/providers/common/compat/lineage/__init__.py

@@ -14,8 +14,3 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-from __future__ import annotations
-
-
-def test_empty():
-    assert True

airflow/providers/common/compat/lineage/hook.py

@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+
+def get_hook_lineage_collector():
+    # HookLineageCollector added in 2.10
+    try:
+        from airflow.lineage.hook import get_hook_lineage_collector
+
+        return get_hook_lineage_collector()
+    except ImportError:
+
+        class NoOpCollector:
+            """
+            NoOpCollector is a hook lineage collector that does nothing.
+
+            It is used when you want to disable lineage collection.
+            """
+
+            def add_input_dataset(self, *_):
+                pass
+
+            def add_output_dataset(self, *_):
+                pass
+
+        return NoOpCollector()

airflow/providers/openlineage/sqlparser.py

@@ -160,7 +160,6 @@ def parse_table_schemas(
             "database": database or database_info.database,
             "use_flat_cross_db_query": database_info.use_flat_cross_db_query,
         }
-        self.log.info("PRE getting schemas for input and output tables")
         return get_table_schemas(
             hook,
             namespace,

contributing-docs/08_static_code_checks.rst

@@ -174,6 +174,8 @@ require Breeze Docker image to be built locally.
 +-----------------------------------------------------------+--------------------------------------------------------------+---------+
 | check-for-inclusive-language                              | Check for language that we do not accept as community        |         |
 +-----------------------------------------------------------+--------------------------------------------------------------+---------+
+| check-get-lineage-collector-providers                     | Check providers import hook lineage code from compat         |         |
++-----------------------------------------------------------+--------------------------------------------------------------+---------+
 | check-google-re2-as-dependency                            | Check google-re2 is declared as dependency when needed       |         |
 +-----------------------------------------------------------+--------------------------------------------------------------+---------+
 | check-hatch-build-order                                   | Check order of dependencies in hatch_build.py                |         |

dev/breeze/doc/images/output_static-checks.txt

@@ -1 +1 @@
-7b121e6337aeb2242ab88b8f51ae1907
+9381c6120248c8e22bd10d9f882ef667

dev/breeze/src/airflow_breeze/pre_commit_ids.py

@@ -52,6 +52,7 @@
     "check-extras-order",
     "check-fab-migrations",
     "check-for-inclusive-language",
+    "check-get-lineage-collector-providers",
     "check-google-re2-as-dependency",
     "check-hatch-build-order",
     "check-hooks-apply",

tests/providers/common/compat/lineage/__init__.py

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

tests/providers/common/compat/lineage/test_hook.py

@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+from airflow.providers.common.compat.lineage.hook import get_hook_lineage_collector
+
+
+def test_that_compat_does_not_raise():
+    # On compat tests this goes into ImportError code path
+    assert get_hook_lineage_collector() is not None