feat: add test for line number errors

Change one csv to get column with line number
Test return rows error are the same based on import from db and csv

Reviewed-by: andriacap

backend/geonature/tests/imports/files/synthese/multiline_comment_file.csv

@@ -1,6 +1,6 @@
-date_min;nom_cite;observateurs;WKT;cd_nom;comment_description
-2017-01-01;Ablette;Toto;POINT(6.5 44.85);;"Ligne erronée : MISSING_VALUE (cd_nom)"
+date_min;nom_cite;observateurs;WKT;cd_nom;comment_description;line_number
+2017-01-01;Ablette;Toto;POINT(6.5 44.85);;"Ligne erronée : MISSING_VALUE (cd_nom)";2
 2017-01-01;Ablette;Toto;POINT(6.5 44.85);123456789;"Ligne erronée :
-CD_NOM_NOT_FOUND"
+CD_NOM_NOT_FOUND";3
 2017-01-01;Ablette;Toto;POINT(6.5 44.85);67111;"Cette ligne
-est valide"
+est valide";4

backend/geonature/tests/imports/test_imports_synthese.py

@@ -39,7 +39,7 @@
 from geonature.core.imports.utils import insert_import_data_in_transient_table
 
 from .jsonschema_definitions import jsonschema_definitions
-from .utils import assert_import_errors
+from .utils import assert_import_errors, extract_row_csv_by_line_number
 
 
 tests_path = Path(__file__).parent
@@ -1235,3 +1235,33 @@ def test_get_nomenclatures(self):
             set(nomenclature.keys()) == {"nomenclature_type", "nomenclatures"}
             for nomenclature in resp.json.values()
         )
+
+    @pytest.mark.parametrize("import_file_name", ["multiline_comment_file.csv"])
+    def test_import_compare_error_line_with_csv(self, users, imported_import, import_file_name):
+        csv_file_path = tests_path / "files" / "synthese" / import_file_name
+        erroneous_rows_line_import = list(map(int, imported_import.erroneous_rows))
+
+        set_logged_user(self.client, users["user"])
+        r = self.client.get(
+            url_for("import.get_import_invalid_rows_as_csv", import_id=imported_import.id_import)
+        )
+        csvfile = StringIO(r.data.decode("utf-8"))
+        csvreader = csv.DictReader(csvfile, delimiter=";")
+        rows_error_imprt = list(csvreader)
+
+        expected_list_rows_error_csv = []
+        for error_line_no in erroneous_rows_line_import:
+            expected_list_rows_error_csv.append(
+                extract_row_csv_by_line_number(csv_file_path, error_line_no)
+            )
+
+        for dict_csv in expected_list_rows_error_csv:
+            assert any(dict_csv == dict_import_err for dict_import_err in rows_error_imprt)
+
+        list_expected_line_number_error = [
+            int(d["line_number"]) for d in expected_list_rows_error_csv if "line_number" in d
+        ]
+        assert all(
+            line_number in erroneous_rows_line_import
+            for line_number in list_expected_line_number_error
+        )

backend/geonature/tests/imports/utils.py

@@ -1,4 +1,5 @@
 from sqlalchemy import or_, select
+import csv
 
 from geonature.utils.env import db
 
@@ -28,3 +29,17 @@ def assert_import_errors(imprt, expected_errors):
         )
         erroneous_rows = {line_no for line_no, in db.session.execute(stmt)}
         assert erroneous_rows == expected_erroneous_rows
+
+
+def extract_row_csv_by_line_number(csv_file_path, line_number, separator=";"):
+
+    with open(csv_file_path, "rb") as file:
+        header_row = file.readline().decode("utf-8").strip().split(separator)
+        file.seek(0)
+        reader = csv.DictReader(
+            (line.decode("utf-8") for line in file), fieldnames=header_row, delimiter=separator
+        )
+        # Start at 1 because idx 0 is header
+        for idx, row in enumerate(reader, start=1):
+            if idx == line_number:
+                return row