test(wip): try to make test working

Reviewed-by: andriacap

backend/geonature/tests/imports/files/synthese/jdd_to_import_file.csv

@@ -0,0 +1,4 @@
+error;id_source;comment_releve;date_debut;date_fin;heure_debut;heure_fin;cd_nom;cd_ref;nom_valide;nom_vernaculaire;nom_cite;regne;group1_inpn;group2_inpn;classe;ordre;famille;rang_taxo;nombre_min;nombre_max;alti_min;alti_max;prof_min;prof_max;observateurs;determinateur;communes;geometrie_wkt_4326;x_centroid_4326;y_centroid_4326;nom_lieu;validateur;niveau_validation;date_validation;comment_validation;preuve_numerique_url;preuve_non_numerique;ca_nom;ca_uuid;ca_id;cd_habref;cd_habitat;nom_habitat;precision_geographique;nature_objet_geo;type_regroupement;methode_regroupement;technique_observation;biologique_statut;etat_biologique;biogeographique_statut;naturalite;preuve_existante;niveau_precision_diffusion;stade_vie;sexe;objet_denombrement;type_denombrement;niveau_sensibilite;statut_observation;floutage_dee;statut_source;type_info_geo;methode_determination;comportement;reference_biblio;uuid_perm_sinp;uuid_perm_grp_sinp;date_creation;date_modification;unique_dataset_id
+valid;1;Relevé n°1;2017-01-01;2017-01-01;12:05:02;12:05:02;60612;60612;Lynx lynx (Linnaeus, 1758);;Lynx Boréal;Animalia;Chordés;Mammifères;Mammalia;Carnivora;Felidae;ES;5;5;1500;1565;;;Administrateur test;Gil;Vallouise-Pelvoux;POINT(6.5 44.85);6.5;44.85;;;En attente de validation;;;;Poil;Données d'observation de la faune, de la Flore et de la fonge du Parc national des Ecrins;57b7d0f2-4183-4b7b-8f08-6e105d476dc5;1;;;;10;Inventoriel;OBS;;Galerie/terrier;Non renseigné;Non renseigné;Non renseigné;Sauvage;Oui;Précise;Adulte;Femelle;Individu;Compté;Non sensible - Diffusion précise;Présent;Non;Terrain;Géoréférencement;Autre méthode de détermination;Non renseigné;;b4f85a2e-dd88-4cdd-aa86-f1c7370faf3f;5b427c76-bd8c-4103-a33c-884c7037aa2b;2021-01-11 14:20:46.492497;2021-01-11 14:20:46.492497;VALID_DATASET_UUID
+valid;1;Relevé n°2;2017-01-01;2017-01-02;12:05:02;12:05:02;351;351;Rana temporaria Linnaeus, 1758;Grenouille rousse (La);Grenouille rousse;Animalia;Chordés;Amphibiens;Amphibia;Anura;Ranidae;ES;1;1;1500;1565;;;Administrateur test;Théo;Vallouise-Pelvoux;POINT(6.5 44.85);6.5;44.85;;;En attente de validation;;;;Poils de plumes;Données d'observation de la faune, de la Flore et de la fonge du Parc national des Ecrins;57b7d0f2-4183-4b7b-8f08-6e105d476dc5;1;;;;10;Inventoriel;OBS;;Galerie/terrier;Non renseigné;Non renseigné;Non renseigné;Sauvage;Oui;Précise;Immature;Femelle;Individu;Compté;Non sensible - Diffusion précise;Présent;Non;Terrain;Géoréférencement;Autre méthode de détermination;Non renseigné;;830c93c7-288e-40f0-a17f-15fbb50e643a;5b427c76-bd8c-4103-a33c-884c7037aa2b;2021-01-11 14:20:46.492497;2021-01-11 14:20:46.492497;
+DATASET_NOT_AUTHORIZED(unique_dataset_id);1;Relevé n°3;2017-01-01;2017-01-01;12:05:02;12:05:02;351;351;Rana temporaria Linnaeus, 1758;Grenouille rousse (La);Grenouille rousse;Animalia;Chordés;Amphibiens;Amphibia;Anura;Ranidae;ES;1;1;1600;1600;;;Administrateur test;Donovan;Vallouise-Pelvoux;POINT(6.5 44.85);6.5;44.85;;;En attente de validation;;;;Poils de plumes;Données d'observation de la faune, de la Flore et de la fonge du Parc national des Ecrins;57b7d0f2-4183-4b7b-8f08-6e105d476dc5;1;;;;100;Inventoriel;OBS;;Galerie/terrier;Non renseigné;Non renseigné;Non renseigné;Sauvage;Oui;Précise;Juvénile;Femelle;Individu;Compté;Non sensible - Diffusion précise;Présent;Non;Terrain;Géoréférencement;Autre méthode de détermination;Non renseigné;;f5515e2a-b30d-11eb-8cc8-af8c2d0867b4;5937d0f2-c96d-424b-bea4-9e3fdac894ed;2021-01-11 14:20:46.492497;2021-01-11 14:20:46.492497;FORBIDDEN_DATASET_UUID

backend/geonature/tests/imports/test_imports_synthese.py

@@ -1,4 +1,4 @@
-from io import StringIO
+from io import StringIO, BytesIO
 from pathlib import Path
 from functools import partial
 from operator import or_
@@ -1271,3 +1271,165 @@ def test_import_compare_error_line_with_csv(self, users, imported_import, import
             assert int(source_row["line_number"]) == erroneous_line_number
             # and this is the test purpose assert:
             assert error_row == source_row
+
+    def test_import_multiple_jdd_file(self, users, datasets):
+        set_logged_user(self.client, users["user"])
+        valid_jdd_file_line_count = 3
+        valid_jdd_file_column_count = 72
+        # Upload step
+        test_file_name = "jdd_to_import_file.csv"
+        with open(tests_path / "files" / "synthese" / test_file_name, "rb") as f:
+            test_file_line_count = sum(1 for line in f) - 1  # remove headers
+            f.seek(0)
+            content = f.read()
+            content = content.replace(
+                b"VALID_DATASET_UUID",
+                datasets["own_dataset"].unique_dataset_id.hex.encode("ascii"),
+            )
+            content = content.replace(
+                b"FORBIDDEN_DATASET_UUID",
+                datasets["orphan_dataset"].unique_dataset_id.hex.encode("ascii"),
+            )
+            f = BytesIO(content)
+
+            data = {
+                "file": (f, test_file_name),
+                "datasetId": datasets["own_dataset"].id_dataset,
+            }
+            r = self.client.post(
+                url_for("import.upload_file"),
+                data=data,
+                headers=Headers({"Content-Type": "multipart/form-data"}),
+            )
+        assert r.status_code == 200, r.data
+        imprt_json = r.get_json()
+        imprt = db.session.get(TImports, imprt_json["id_import"])
+        assert len(imprt.authors) == 1
+        assert imprt_json["date_create_import"]
+        assert imprt_json["date_update_import"]
+        assert imprt_json["detected_encoding"] == "utf-8"
+        assert imprt_json["detected_format"] == "csv"
+        assert imprt_json["detected_separator"] == ";"
+        assert imprt_json["full_file_name"] == test_file_name
+        assert imprt_json["id_dataset"] == datasets["own_dataset"].id_dataset
+
+        # Decode step
+        data = {
+            "encoding": "utf-8",
+            "format": "csv",
+            "srid": 4326,
+            "separator": ";",
+        }
+        r = self.client.post(url_for("import.decode_file", import_id=imprt.id_import), data=data)
+        assert r.status_code == 200, r.data
+        validate_json(
+            r.json,
+            {"definitions": jsonschema_definitions, "$ref": "#/definitions/import"},
+        )
+        assert imprt.date_update_import
+        assert imprt.encoding == "utf-8"
+        assert imprt.format_source_file == "csv"
+        assert imprt.separator == ";"
+        assert imprt.srid == 4326
+        assert imprt.columns
+        assert len(imprt.columns) == valid_jdd_file_column_count
+        transient_table = imprt.destination.get_transient_table()
+        transient_rows_count = db.session.execute(
+            select([func.count()])
+            .select_from(transient_table)
+            .where(transient_table.c.id_import == imprt.id_import)
+        ).scalar()
+        assert transient_rows_count == 0
+
+        # Field mapping step
+        fieldmapping = FieldMapping.query.filter_by(label="Synthese GeoNature").one()
+        r = self.client.post(
+            url_for("import.set_import_field_mapping", import_id=imprt.id_import),
+            data=fieldmapping.values,
+        )
+        assert r.status_code == 200, r.data
+        validate_json(
+            r.json,
+            {"definitions": jsonschema_definitions, "$ref": "#/definitions/import"},
+        )
+        assert r.json["fieldmapping"] == fieldmapping.values
+
+        # Loading step
+        r = self.client.post(url_for("import.load_import", import_id=imprt.id_import))
+        assert r.status_code == 200, r.data
+        assert r.json["source_count"] == valid_jdd_file_line_count
+        assert imprt.source_count == valid_jdd_file_line_count
+        assert imprt.loaded == True
+        transient_rows_count = db.session.execute(
+            select([func.count()])
+            .select_from(transient_table)
+            .where(transient_table.c.id_import == imprt.id_import)
+        ).scalar()
+        assert transient_rows_count == test_file_line_count
+
+        # Content mapping step
+        contentmapping = ContentMapping.query.filter_by(label="Nomenclatures SINP (labels)").one()
+        r = self.client.post(
+            url_for("import.set_import_content_mapping", import_id=imprt.id_import),
+            data=contentmapping.values,
+        )
+        assert r.status_code == 200, r.data
+        data = r.get_json()
+        validate_json(
+            data,
+            {"definitions": jsonschema_definitions, "$ref": "#/definitions/import"},
+        )
+        assert data["contentmapping"] == contentmapping.values
+
+        # Prepare data before import
+        r = self.client.post(url_for("import.prepare_import", import_id=imprt.id_import))
+        assert r.status_code == 200, r.data
+        validate_json(
+            r.json,
+            {"definitions": jsonschema_definitions, "$ref": "#/definitions/import"},
+        )
+        assert_import_errors(imprt, valid_file_expected_errors)
+
+        # Get errors
+        r = self.client.get(url_for("import.get_import_errors", import_id=imprt.id_import))
+        assert r.status_code == 200, r.data
+        assert len(r.json) == len(valid_file_expected_errors)
+
+        # Get valid data (preview)
+        r = self.client.get(url_for("import.preview_valid_data", import_id=imprt.id_import))
+        assert r.status_code == 200, r.data
+        assert r.json["entities"][0]["n_valid_data"] == imprt.source_count - len(
+            valid_file_invalid_rows
+        )
+        assert r.json["entities"][0]["n_invalid_data"] == len(valid_file_invalid_rows)
+
+        # Get invalid data
+        # The with block forcefully close the request context, which may stay open due
+        # to the usage of stream_with_context in this route.
+        with self.client.get(
+            url_for("import.get_import_invalid_rows_as_csv", import_id=imprt.id_import)
+        ) as r:
+            assert r.status_code == 200, r.data
+
+        # Import step
+        r = self.client.post(url_for("import.import_valid_data", import_id=imprt.id_import))
+        assert r.status_code == 200, r.data
+        data = r.get_json()
+        validate_json(
+            data,
+            {"definitions": jsonschema_definitions, "$ref": "#/definitions/import"},
+        )
+        transient_rows_count = db.session.execute(
+            select([func.count()])
+            .select_from(transient_table)
+            .where(transient_table.c.id_import == imprt.id_import)
+        ).scalar()
+        assert transient_rows_count == 0
+        assert valid_jdd_file_line_count - len(valid_file_invalid_rows) == imprt.import_count
+        assert valid_file_taxa_count == imprt.statistics["taxa_count"]
+        source = TSources.query.filter_by(name_source=f"Import(id={imprt.id_import})").one()
+        assert Synthese.query.filter_by(source=source).count() == imprt.import_count
+
+        # Delete step
+        r = self.client.delete(url_for("import.delete_import", import_id=imprt.id_import))
+        assert r.status_code == 200, r.data