A

setup architecture design Co-authored-by: Leon Moll<[email protected]>

Signed-off-by: Christian Munz <[email protected]>

src/sdk/python/rtdip_sdk/pipelines/data_wranglers/__init__.py

@@ -0,0 +1,14 @@
+# Copyright 2022 RTDIP
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from .spark.data_quality.duplicate_detection import *

src/sdk/python/rtdip_sdk/pipelines/data_wranglers/interfaces.py

@@ -0,0 +1,22 @@
+# Copyright 2022 RTDIP
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import abstractmethod
+from ..interfaces import PipelineComponentBaseInterface
+
+
+class WranglerBaseInterface(PipelineComponentBaseInterface):
+    @abstractmethod
+    def filter(self):
+        pass

src/sdk/python/rtdip_sdk/pipelines/data_wranglers/spark/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2022 RTDIP
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

src/sdk/python/rtdip_sdk/pipelines/data_wranglers/spark/data_quality/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2022 RTDIP
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

src/sdk/python/rtdip_sdk/pipelines/monitoring/spark/data_quality/duplicate_detection.py → src/sdk/python/rtdip_sdk/pipelines/data_wranglers/spark/data_quality/duplicate_detection.py

@@ -14,8 +14,8 @@
 from pyspark.sql.functions import desc
 from pyspark.sql import DataFrame as PySparkDataFrame
 
-from ...interfaces import MonitoringBaseInterface
-from ...._pipeline_utils.models import Libraries, SystemType
+from rtdip_sdk.pipelines.monitoring.interfaces import MonitoringBaseInterface
+from rtdip_sdk.pipelines._pipeline_utils.models import Libraries, SystemType
 
 
 class DuplicateDetection(MonitoringBaseInterface):

src/sdk/python/rtdip_sdk/pipelines/monitoring/__init__.py

@@ -12,4 +12,3 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 from .spark.data_quality.great_expectations_data_quality import *
-from .spark.data_quality.duplicate_detection import *

src/sdk/python/rtdip_sdk/pipelines/monitoring/interfaces.py

@@ -17,4 +17,6 @@
 
 
 class MonitoringBaseInterface(PipelineComponentBaseInterface):
-    pass
+    @abstractmethod
+    def check(self):
+        pass

tests/sdk/python/rtdip_sdk/pipelines/monitoring/spark/data_quality/test_duplicate_detection.py

@@ -16,7 +16,7 @@
 from pyspark.sql import SparkSession
 from pyspark.sql.dataframe import DataFrame
 
-from src.sdk.python.rtdip_sdk.pipelines.monitoring.spark.data_quality.duplicate_detection import DuplicateDetection
+from rtdip_sdk.pipelines.data_wranglers.spark.data_quality.duplicate_detection import DuplicateDetection
 
 
 @pytest.fixture(scope="session")