fix(test): errors with unique_dataset_id

Fix errors based on adding unique_dataset_id

Reviewed-by: andriacap

backend/geonature/core/gn_synthese/imports/__init__.py

@@ -91,19 +91,6 @@ def update_batch_progress(batch, step):
             )
         update_batch_progress(batch, 1)
 
-        logger.info(f"[{batch+1}/{batch_count}] Check dataset rows")
-        with start_sentry_child(op="check.df", description="check datasets rows"):
-            updated_cols |= check_datasets(
-                imprt,
-                entity,
-                df,
-                uuid_field=fields["unique_dataset_id"],
-                id_field=fields["id_dataset"],
-                module_code="SYNTHESE",
-            )
-
-        update_batch_progress(batch, 2)
-
         logger.info(f"[{batch+1}/{batch_count}] Concat dates…")
         with start_sentry_child(op="check.df", description="concat dates"):
             updated_cols |= concat_dates(
@@ -115,16 +102,28 @@ def update_batch_progress(batch, step):
                 fields["hour_min"].source_field,
                 fields["hour_max"].source_field,
             )
-        update_batch_progress(batch, 3)
+        update_batch_progress(batch, 2)
 
         logger.info(f"[{batch+1}/{batch_count}] Check required values…")
         with start_sentry_child(op="check.df", description="check required values"):
             updated_cols |= check_required_values(imprt, entity, df, fields)
-        update_batch_progress(batch, 4)
+        update_batch_progress(batch, 3)
 
         logger.info(f"[{batch+1}/{batch_count}] Check types…")
         with start_sentry_child(op="check.df", description="check types"):
             updated_cols |= check_types(imprt, entity, df, fields)
+
+        update_batch_progress(batch, 4)
+        logger.info(f"[{batch+1}/{batch_count}] Check dataset rows")
+        with start_sentry_child(op="check.df", description="check datasets rows"):
+            updated_cols |= check_datasets(
+                imprt,
+                entity,
+                df,
+                uuid_field=fields["unique_dataset_id"],
+                id_field=fields["id_dataset"],
+                module_code="SYNTHESE",
+            )
         update_batch_progress(batch, 5)
 
         logger.info(f"[{batch+1}/{batch_count}] Check geography…")
@@ -320,6 +319,7 @@ def import_data_to_synthese(imprt):
             if source_field in imprt.columns:
                 insert_fields |= {field}
 
+    insert_fields -= {fields["unique_dataset_id"]}
     select_stmt = (
         sa.select(
             *[transient_table.c[field.dest_field] for field in insert_fields],

backend/geonature/tests/imports/files/synthese/jdd_to_import_file.csv

@@ -0,0 +1,5 @@
+error;id_synthese;id_origine;comment_releve;comment_occurrence;date_debut;date_fin;heure_debut;heure_fin;cd_nom;cd_ref;nom_valide;nom_vernaculaire;nom_cite;regne;group1_inpn;group2_inpn;classe;ordre;famille;rang_taxo;nombre_min;nombre_max;alti_min;alti_max;prof_min;prof_max;observateurs;determinateur;communes;geometrie_wkt_4326;x_centroid_4326;y_centroid_4326;nom_lieu;validateur;niveau_validation;date_validation;comment_validation;preuve_numerique_url;preuve_non_numerique;jdd_nom;jdd_uuid;jdd_id;ca_nom;ca_uuid;ca_id;cd_habref;cd_habitat;nom_habitat;precision_geographique;nature_objet_geo;type_regroupement;methode_regroupement;technique_observation;biologique_statut;etat_biologique;biogeographique_statut;naturalite;preuve_existante;niveau_precision_diffusion;stade_vie;sexe;objet_denombrement;type_denombrement;niveau_sensibilite;statut_observation;floutage_dee;statut_source;type_info_geo;methode_determination;comportement;reference_biblio;uuid_perm_sinp;uuid_perm_grp_sinp;date_creation;date_modification;unique_dataset_id
+valid;1;1;Relevé n°1;Occurrence n°1;2017-01-01;2017-01-01;12:05:02;12:05:02;60612;60612;Lynx lynx (Linnaeus, 1758);;Lynx Boréal;Animalia;Chordés;Mammifères;Mammalia;Carnivora;Felidae;ES;5;5;1500;1565;;;Administrateur test;Gil;Vallouise-Pelvoux;POINT(6.5 44.85);6.5;44.85;;;En attente de validation;;;;Poil;Contact aléatoire tous règnes confondus;4d331cae-65e4-4948-b0b2-a11bc5bb46c2;1;Données d'observation de la faune, de la Flore et de la fonge du Parc national des Ecrins;57b7d0f2-4183-4b7b-8f08-6e105d476dc5;1;;;;10;Inventoriel;OBS;;Galerie/terrier;Non renseigné;Non renseigné;Non renseigné;Sauvage;Oui;Précise;Adulte;Femelle;Individu;Compté;Non sensible - Diffusion précise;Présent;Non;Terrain;Géoréférencement;Autre méthode de détermination;Non renseigné;;b4f85a2e-dd88-4cdd-aa86-f1c7370faf3f;5b427c76-bd8c-4103-a33c-884c7037aa2b;2021-01-11 14:20:46.492497;2021-01-11 14:20:46.492497;VALID_DATASET_UUID
+valid;2;2;Relevé n°2;Occurrence n°2;2017-01-01;2017-01-02;12:05:02;12:05:02;351;351;Rana temporaria Linnaeus, 1758;Grenouille rousse (La);Grenouille rousse;Animalia;Chordés;Amphibiens;Amphibia;Anura;Ranidae;ES;1;1;1500;1565;;;Administrateur test;Théo;Vallouise-Pelvoux;POINT(6.5 44.85);6.5;44.85;;;En attente de validation;;;;Poils de plumes;Contact aléatoire tous règnes confondus;4d331cae-65e4-4948-b0b2-a11bc5bb46c2;1;Données d'observation de la faune, de la Flore et de la fonge du Parc national des Ecrins;57b7d0f2-4183-4b7b-8f08-6e105d476dc5;1;;;;10;Inventoriel;OBS;;Galerie/terrier;Non renseigné;Non renseigné;Non renseigné;Sauvage;Oui;Précise;Immature;Femelle;Individu;Compté;Non sensible - Diffusion précise;Présent;Non;Terrain;Géoréférencement;Autre méthode de détermination;Non renseigné;;830c93c7-288e-40f0-a17f-15fbb50e643a;5b427c76-bd8c-4103-a33c-884c7037aa2b;2021-01-11 14:20:46.492497;2021-01-11 14:20:46.492497;
+DATASET_NOT_AUTHORIZED(unique_dataset_id);3;3;Relevé n°3;Occurrence n°3;2017-01-08;;;;67111;67111;Alburnus alburnus (Linnaeus, 1758);Ablette;Ablette;Animalia;Chordés;Poissons;Actinopterygii;Cypriniformes;Leuciscidae;ES;1;1;1600;1600;;;Administrateur test;Donovan;Vallouise-Pelvoux;POINT(6.5 44.85);6.5;44.85;;;En attente de validation;;;;Poils de plumes;Contact aléatoire tous règnes confondus;4d331cae-65e4-4948-b0b2-a11bc5bb46c2;1;Données d'observation de la faune, de la Flore et de la fonge du Parc national des Ecrins;57b7d0f2-4183-4b7b-8f08-6e105d476dc5;1;;;;100;Inventoriel;OBS;;Galerie/terrier;Non renseigné;Non renseigné;Non renseigné;Sauvage;Oui;Précise;Juvénile;Femelle;Individu;Compté;Non sensible - Diffusion précise;Présent;Non;Terrain;Géoréférencement;Autre méthode de détermination;Non renseigné;;2f92f91a-64a2-4684-90e4-140466bb34e3;5937d0f2-c96d-424b-bea4-9e3fdac894ed;2021-01-11 14:20:46.492497;2021-01-11 14:20:46.492497;FORBIDDEN_DATASET_UUID
+INVALID_UUID(unique_dataset_id);6;6;Relevé n°6;Occurrence n°6;2017-01-01;2017-01-01;12:05:02;12:05:02;351;351;Rana temporaria Linnaeus, 1758;Grenouille rousse (La);Grenouille rousse;Animalia;Chordés;Amphibiens;Amphibia;Anura;Ranidae;ES;1;1;1600;1600;;;Administrateur test;Donovan;Vallouise-Pelvoux;POINT(6.5 44.85);6.5;44.85;;;En attente de validation;;;;Poils de plumes;Contact aléatoire tous règnes confondus;4d331cae-65e4-4948-b0b2-a11bc5bb46c2;1;Données d'observation de la faune, de la Flore et de la fonge du Parc national des Ecrins;57b7d0f2-4183-4b7b-8f08-6e105d476dc5;1;;;;100;Inventoriel;OBS;;Galerie/terrier;Non renseigné;Non renseigné;Non renseigné;Sauvage;Oui;Précise;Juvénile;Femelle;Individu;Compté;Non sensible - Diffusion précise;Présent;Non;Terrain;Géoréférencement;Autre méthode de détermination;Non renseigné;;f5515e2a-b30d-11eb-8cc8-af8c2d0867b4;5937d0f2-c96d-424b-bea4-9e3fdac894ed;2021-01-11 14:20:46.492497;2021-01-11 14:20:46.492497;050d613c-543f-47fd-800a-13931b2721c7

backend/geonature/tests/imports/test_imports_synthese.py

@@ -1,4 +1,4 @@
-from io import StringIO
+from io import StringIO, BytesIO
 from pathlib import Path
 from functools import partial
 from operator import or_
@@ -143,11 +143,25 @@ def new_import(synthese_destination, users, import_dataset):
 
 
 @pytest.fixture()
-def uploaded_import(new_import, import_file_name):
+def uploaded_import(client, new_import, datasets, import_file_name):
     with db.session.begin_nested():
         with open(tests_path / "files" / "synthese" / import_file_name, "rb") as f:
-            new_import.source_file = f.read()
-            new_import.full_file_name = "valid_file.csv"
+            f.seek(0)
+            content = f.read()
+            if import_file_name == "jdd_to_import_file.csv":
+                content = content.replace(
+                    b"VALID_DATASET_UUID",
+                    datasets["own_dataset"].unique_dataset_id.hex.encode("ascii"),
+                )
+                content = content.replace(
+                    b"FORBIDDEN_DATASET_UUID",
+                    datasets["orphan_dataset"].unique_dataset_id.hex.encode("ascii"),
+                )
+                new_import.full_file_name = "jdd_to_import_file.csv"
+            else:
+                new_import.full_file_name = "valid_file.csv"
+            new_import.source_file = content
+            # data = bytes_csv_to_dict(content)
     return new_import
 
 
@@ -174,7 +188,7 @@ def decoded_import(client, uploaded_import):
 
 @pytest.fixture()
 def fieldmapping(import_file_name, autogenerate):
-    if import_file_name == "valid_file.csv":
+    if import_file_name in ["valid_file.csv", "jdd_to_import_file.csv"]:
         return FieldMapping.query.filter_by(label="Synthese GeoNature").one().values
     else:
         return {
@@ -206,16 +220,27 @@ def loaded_import(client, field_mapped_import):
 
 @pytest.fixture()
 def content_mapped_import(client, import_file_name, loaded_import):
-    with db.session.begin_nested():
-        loaded_import.contentmapping = (
-            ContentMapping.query.filter_by(label="Nomenclatures SINP (labels)").one().values
+    if import_file_name == "jdd_to_import_file.csv":
+        set_logged_user(client, loaded_import.authors[0])
+        contentmapping = ContentMapping.query.filter_by(label="Nomenclatures SINP (labels)").one()
+        r = client.post(
+            url_for("import.set_import_content_mapping", import_id=loaded_import.id_import),
+            data=contentmapping.values,
         )
-        if import_file_name == "empty_nomenclatures_file.csv":
-            loaded_import.contentmapping["STADE_VIE"].update(
-                {
-                    "": "17",  # Alevin
-                }
+        assert r.status_code == 200, r.data
+        unset_logged_user(client)
+        db.session.refresh(loaded_import)
+    else:
+        with db.session.begin_nested():
+            loaded_import.contentmapping = (
+                ContentMapping.query.filter_by(label="Nomenclatures SINP (labels)").one().values
             )
+            if import_file_name == "empty_nomenclatures_file.csv":
+                loaded_import.contentmapping["STADE_VIE"].update(
+                    {
+                        "": "17",  # Alevin
+                    }
+                )
     return loaded_import
 
 
@@ -1271,3 +1296,14 @@ def test_import_compare_error_line_with_csv(self, users, imported_import, import
             assert int(source_row["line_number"]) == erroneous_line_number
             # and this is the test purpose assert:
             assert error_row == source_row
+
+    @pytest.mark.parametrize("import_file_name", ["jdd_to_import_file.csv"])
+    def test_import_jdd_file(self, imported_import):
+        assert_import_errors(
+            imported_import,
+            {
+                # id_dataset errors
+                ("DATASET_NOT_FOUND", "unique_dataset_id", frozenset({5})),
+                ("DATASET_NOT_AUTHORIZED", "unique_dataset_id", frozenset({4})),
+            },
+        )