diff --git a/backend/geonature/tests/imports/files/synthese/jdd_to_import_file.csv b/backend/geonature/tests/imports/files/synthese/jdd_to_import_file.csv new file mode 100644 index 0000000000..35e860cece --- /dev/null +++ b/backend/geonature/tests/imports/files/synthese/jdd_to_import_file.csv @@ -0,0 +1,4 @@ +error;id_source;comment_releve;date_debut;date_fin;heure_debut;heure_fin;cd_nom;cd_ref;nom_valide;nom_vernaculaire;nom_cite;regne;group1_inpn;group2_inpn;classe;ordre;famille;rang_taxo;nombre_min;nombre_max;alti_min;alti_max;prof_min;prof_max;observateurs;determinateur;communes;geometrie_wkt_4326;x_centroid_4326;y_centroid_4326;nom_lieu;validateur;niveau_validation;date_validation;comment_validation;preuve_numerique_url;preuve_non_numerique;ca_nom;ca_uuid;ca_id;cd_habref;cd_habitat;nom_habitat;precision_geographique;nature_objet_geo;type_regroupement;methode_regroupement;technique_observation;biologique_statut;etat_biologique;biogeographique_statut;naturalite;preuve_existante;niveau_precision_diffusion;stade_vie;sexe;objet_denombrement;type_denombrement;niveau_sensibilite;statut_observation;floutage_dee;statut_source;type_info_geo;methode_determination;comportement;reference_biblio;uuid_perm_sinp;uuid_perm_grp_sinp;date_creation;date_modification;unique_dataset_id +valid;1;Relevé n°1;2017-01-01;2017-01-01;12:05:02;12:05:02;60612;60612;Lynx lynx (Linnaeus, 1758);;Lynx Boréal;Animalia;Chordés;Mammifères;Mammalia;Carnivora;Felidae;ES;5;5;1500;1565;;;Administrateur test;Gil;Vallouise-Pelvoux;POINT(6.5 44.85);6.5;44.85;;;En attente de validation;;;;Poil;Données d'observation de la faune, de la Flore et de la fonge du Parc national des Ecrins;57b7d0f2-4183-4b7b-8f08-6e105d476dc5;1;;;;10;Inventoriel;OBS;;Galerie/terrier;Non renseigné;Non renseigné;Non renseigné;Sauvage;Oui;Précise;Adulte;Femelle;Individu;Compté;Non sensible - Diffusion précise;Présent;Non;Terrain;Géoréférencement;Autre méthode de détermination;Non renseigné;;b4f85a2e-dd88-4cdd-aa86-f1c7370faf3f;5b427c76-bd8c-4103-a33c-884c7037aa2b;2021-01-11 14:20:46.492497;2021-01-11 14:20:46.492497;VALID_DATASET_UUID +valid;1;Relevé n°2;2017-01-01;2017-01-02;12:05:02;12:05:02;351;351;Rana temporaria Linnaeus, 1758;Grenouille rousse (La);Grenouille rousse;Animalia;Chordés;Amphibiens;Amphibia;Anura;Ranidae;ES;1;1;1500;1565;;;Administrateur test;Théo;Vallouise-Pelvoux;POINT(6.5 44.85);6.5;44.85;;;En attente de validation;;;;Poils de plumes;Données d'observation de la faune, de la Flore et de la fonge du Parc national des Ecrins;57b7d0f2-4183-4b7b-8f08-6e105d476dc5;1;;;;10;Inventoriel;OBS;;Galerie/terrier;Non renseigné;Non renseigné;Non renseigné;Sauvage;Oui;Précise;Immature;Femelle;Individu;Compté;Non sensible - Diffusion précise;Présent;Non;Terrain;Géoréférencement;Autre méthode de détermination;Non renseigné;;830c93c7-288e-40f0-a17f-15fbb50e643a;5b427c76-bd8c-4103-a33c-884c7037aa2b;2021-01-11 14:20:46.492497;2021-01-11 14:20:46.492497; +DATASET_NOT_AUTHORIZED(unique_dataset_id);1;Relevé n°3;2017-01-01;2017-01-01;12:05:02;12:05:02;351;351;Rana temporaria Linnaeus, 1758;Grenouille rousse (La);Grenouille rousse;Animalia;Chordés;Amphibiens;Amphibia;Anura;Ranidae;ES;1;1;1600;1600;;;Administrateur test;Donovan;Vallouise-Pelvoux;POINT(6.5 44.85);6.5;44.85;;;En attente de validation;;;;Poils de plumes;Données d'observation de la faune, de la Flore et de la fonge du Parc national des Ecrins;57b7d0f2-4183-4b7b-8f08-6e105d476dc5;1;;;;100;Inventoriel;OBS;;Galerie/terrier;Non renseigné;Non renseigné;Non renseigné;Sauvage;Oui;Précise;Juvénile;Femelle;Individu;Compté;Non sensible - Diffusion précise;Présent;Non;Terrain;Géoréférencement;Autre méthode de détermination;Non renseigné;;f5515e2a-b30d-11eb-8cc8-af8c2d0867b4;5937d0f2-c96d-424b-bea4-9e3fdac894ed;2021-01-11 14:20:46.492497;2021-01-11 14:20:46.492497;FORBIDDEN_DATASET_UUID diff --git a/backend/geonature/tests/imports/test_imports_synthese.py b/backend/geonature/tests/imports/test_imports_synthese.py index f080cd2649..9921d8b481 100644 --- a/backend/geonature/tests/imports/test_imports_synthese.py +++ b/backend/geonature/tests/imports/test_imports_synthese.py @@ -1,4 +1,4 @@ -from io import StringIO +from io import StringIO, BytesIO from pathlib import Path from functools import partial from operator import or_ @@ -1271,3 +1271,165 @@ def test_import_compare_error_line_with_csv(self, users, imported_import, import assert int(source_row["line_number"]) == erroneous_line_number # and this is the test purpose assert: assert error_row == source_row + + def test_import_multiple_jdd_file(self, users, datasets): + set_logged_user(self.client, users["user"]) + valid_jdd_file_line_count = 3 + valid_jdd_file_column_count = 72 + # Upload step + test_file_name = "jdd_to_import_file.csv" + with open(tests_path / "files" / "synthese" / test_file_name, "rb") as f: + test_file_line_count = sum(1 for line in f) - 1 # remove headers + f.seek(0) + content = f.read() + content = content.replace( + b"VALID_DATASET_UUID", + datasets["own_dataset"].unique_dataset_id.hex.encode("ascii"), + ) + content = content.replace( + b"FORBIDDEN_DATASET_UUID", + datasets["orphan_dataset"].unique_dataset_id.hex.encode("ascii"), + ) + f = BytesIO(content) + + data = { + "file": (f, test_file_name), + "datasetId": datasets["own_dataset"].id_dataset, + } + r = self.client.post( + url_for("import.upload_file"), + data=data, + headers=Headers({"Content-Type": "multipart/form-data"}), + ) + assert r.status_code == 200, r.data + imprt_json = r.get_json() + imprt = db.session.get(TImports, imprt_json["id_import"]) + assert len(imprt.authors) == 1 + assert imprt_json["date_create_import"] + assert imprt_json["date_update_import"] + assert imprt_json["detected_encoding"] == "utf-8" + assert imprt_json["detected_format"] == "csv" + assert imprt_json["detected_separator"] == ";" + assert imprt_json["full_file_name"] == test_file_name + assert imprt_json["id_dataset"] == datasets["own_dataset"].id_dataset + + # Decode step + data = { + "encoding": "utf-8", + "format": "csv", + "srid": 4326, + "separator": ";", + } + r = self.client.post(url_for("import.decode_file", import_id=imprt.id_import), data=data) + assert r.status_code == 200, r.data + validate_json( + r.json, + {"definitions": jsonschema_definitions, "$ref": "#/definitions/import"}, + ) + assert imprt.date_update_import + assert imprt.encoding == "utf-8" + assert imprt.format_source_file == "csv" + assert imprt.separator == ";" + assert imprt.srid == 4326 + assert imprt.columns + assert len(imprt.columns) == valid_jdd_file_column_count + transient_table = imprt.destination.get_transient_table() + transient_rows_count = db.session.execute( + select([func.count()]) + .select_from(transient_table) + .where(transient_table.c.id_import == imprt.id_import) + ).scalar() + assert transient_rows_count == 0 + + # Field mapping step + fieldmapping = FieldMapping.query.filter_by(label="Synthese GeoNature").one() + r = self.client.post( + url_for("import.set_import_field_mapping", import_id=imprt.id_import), + data=fieldmapping.values, + ) + assert r.status_code == 200, r.data + validate_json( + r.json, + {"definitions": jsonschema_definitions, "$ref": "#/definitions/import"}, + ) + assert r.json["fieldmapping"] == fieldmapping.values + + # Loading step + r = self.client.post(url_for("import.load_import", import_id=imprt.id_import)) + assert r.status_code == 200, r.data + assert r.json["source_count"] == valid_jdd_file_line_count + assert imprt.source_count == valid_jdd_file_line_count + assert imprt.loaded == True + transient_rows_count = db.session.execute( + select([func.count()]) + .select_from(transient_table) + .where(transient_table.c.id_import == imprt.id_import) + ).scalar() + assert transient_rows_count == test_file_line_count + + # Content mapping step + contentmapping = ContentMapping.query.filter_by(label="Nomenclatures SINP (labels)").one() + r = self.client.post( + url_for("import.set_import_content_mapping", import_id=imprt.id_import), + data=contentmapping.values, + ) + assert r.status_code == 200, r.data + data = r.get_json() + validate_json( + data, + {"definitions": jsonschema_definitions, "$ref": "#/definitions/import"}, + ) + assert data["contentmapping"] == contentmapping.values + + # Prepare data before import + r = self.client.post(url_for("import.prepare_import", import_id=imprt.id_import)) + assert r.status_code == 200, r.data + validate_json( + r.json, + {"definitions": jsonschema_definitions, "$ref": "#/definitions/import"}, + ) + assert_import_errors(imprt, valid_file_expected_errors) + + # Get errors + r = self.client.get(url_for("import.get_import_errors", import_id=imprt.id_import)) + assert r.status_code == 200, r.data + assert len(r.json) == len(valid_file_expected_errors) + + # Get valid data (preview) + r = self.client.get(url_for("import.preview_valid_data", import_id=imprt.id_import)) + assert r.status_code == 200, r.data + assert r.json["entities"][0]["n_valid_data"] == imprt.source_count - len( + valid_file_invalid_rows + ) + assert r.json["entities"][0]["n_invalid_data"] == len(valid_file_invalid_rows) + + # Get invalid data + # The with block forcefully close the request context, which may stay open due + # to the usage of stream_with_context in this route. + with self.client.get( + url_for("import.get_import_invalid_rows_as_csv", import_id=imprt.id_import) + ) as r: + assert r.status_code == 200, r.data + + # Import step + r = self.client.post(url_for("import.import_valid_data", import_id=imprt.id_import)) + assert r.status_code == 200, r.data + data = r.get_json() + validate_json( + data, + {"definitions": jsonschema_definitions, "$ref": "#/definitions/import"}, + ) + transient_rows_count = db.session.execute( + select([func.count()]) + .select_from(transient_table) + .where(transient_table.c.id_import == imprt.id_import) + ).scalar() + assert transient_rows_count == 0 + assert valid_jdd_file_line_count - len(valid_file_invalid_rows) == imprt.import_count + assert valid_file_taxa_count == imprt.statistics["taxa_count"] + source = TSources.query.filter_by(name_source=f"Import(id={imprt.id_import})").one() + assert Synthese.query.filter_by(source=source).count() == imprt.import_count + + # Delete step + r = self.client.delete(url_for("import.delete_import", import_id=imprt.id_import)) + assert r.status_code == 200, r.data