diff --git a/backend/geonature/core/gn_synthese/imports/actions.py b/backend/geonature/core/gn_synthese/imports/actions.py index fc501b1d53..5284932c24 100644 --- a/backend/geonature/core/gn_synthese/imports/actions.py +++ b/backend/geonature/core/gn_synthese/imports/actions.py @@ -88,7 +88,7 @@ def check_transient_data(task, logger, imprt: TImports): selected_fields = { field_name: fields[field_name] for field_name, source_field in imprt.fieldmapping.items() - if source_field in imprt.columns + if source_field.get('column_src', None) in imprt.columns } init_rows_validity(imprt) task.update_state(state="PROGRESS", meta={"progress": 0.05}) @@ -339,11 +339,12 @@ def import_data_to_destination(imprt: TImports) -> None: if field_name not in fields: # not a destination field continue field = fields[field_name] + column_src = source_field.get('column_src', None) if field.multi: - if not set(source_field).isdisjoint(imprt.columns): + if not set(column_src).isdisjoint(imprt.columns): insert_fields |= {field} else: - if source_field in imprt.columns: + if column_src in imprt.columns: insert_fields |= {field} insert_fields -= {fields["unique_dataset_id"]} # Column only used for filling `id_dataset` diff --git a/backend/geonature/core/imports/checks/dataframe/utils.py b/backend/geonature/core/imports/checks/dataframe/utils.py index 4257442559..6dff9ff6d1 100644 --- a/backend/geonature/core/imports/checks/dataframe/utils.py +++ b/backend/geonature/core/imports/checks/dataframe/utils.py @@ -7,7 +7,7 @@ from geonature.utils.env import db -from geonature.core.imports.models import ImportUserError, ImportUserErrorType +from geonature.core.imports.models import ImportUserError, ImportUserErrorType, TImports from geonature.core.imports.utils import generated_fields @@ -101,7 +101,7 @@ def __error_replace(*args, **kwargs): return _error_replace -def report_error(imprt, entity, df, error): +def report_error(imprt: TImports, entity, df, error): """ Reports an error found in the dataframe, updates the validity column and insert the error in the `t_user_errors` table. @@ -147,7 +147,7 @@ def report_error(imprt, entity, df, error): # f'{error_type.name}' # FIXME comment ordered_invalid_rows = sorted(invalid_rows["line_no"]) column = generated_fields.get(error["column"], error["column"]) - column = imprt.fieldmapping.get(column, column) + column = imprt.fieldmapping.get(column, {}).get("column_src", column) # If an error for same import, same column and of the same type already exists, # we concat existing erroneous rows with current rows. stmt = pg_insert(ImportUserError).values( diff --git a/backend/geonature/core/imports/checks/sql/core.py b/backend/geonature/core/imports/checks/sql/core.py index c6fe31d26b..ad2d6a52dc 100644 --- a/backend/geonature/core/imports/checks/sql/core.py +++ b/backend/geonature/core/imports/checks/sql/core.py @@ -8,13 +8,14 @@ Entity, EntityField, BibFields, + TImports, ) __all__ = ["init_rows_validity", "check_orphan_rows"] -def init_rows_validity(imprt): +def init_rows_validity(imprt: TImports): """ Validity columns are three-states: - None: the row does not contains data for the given entity @@ -35,9 +36,10 @@ def init_rows_validity(imprt): # as rows with multi-entity field only will raise an ORPHAN_ROW error selected_fields_names = [] for field_name, source_field in imprt.fieldmapping.items(): - if type(source_field) == list: - selected_fields_names.extend(set(source_field) & set(imprt.columns)) - elif source_field in imprt.columns: + column_src = source_field.get("column_src", None) + if type(column_src) == list: + selected_fields_names.extend(set(column_src) & set(imprt.columns)) + elif column_src in imprt.columns: selected_fields_names.append(field_name) for entity in entities: # Select fields associated to this entity *and only to this entity* @@ -58,15 +60,16 @@ def init_rows_validity(imprt): ) -def check_orphan_rows(imprt): +def check_orphan_rows(imprt: TImports): transient_table = imprt.destination.get_transient_table() # TODO: handle multi-source fields # This is actually not a big issue as multi-source fields are unlikely to also be multi-entity fields. selected_fields_names = [] for field_name, source_field in imprt.fieldmapping.items(): - if type(source_field) == list: - selected_fields_names.extend(set(source_field) & set(imprt.columns)) - elif source_field in imprt.columns: + column_src = source_field.get("column_src", None) + if type(column_src) == list: + selected_fields_names.extend(set(column_src) & set(imprt.columns)) + elif column_src in imprt.columns: selected_fields_names.append(field_name) # Select fields associated to multiple entities AllEntityField = sa.orm.aliased(EntityField) diff --git a/backend/geonature/core/imports/checks/sql/utils.py b/backend/geonature/core/imports/checks/sql/utils.py index e07168b001..403cbca4f1 100644 --- a/backend/geonature/core/imports/checks/sql/utils.py +++ b/backend/geonature/core/imports/checks/sql/utils.py @@ -64,7 +64,7 @@ def report_erroneous_rows( transient_table = imprt.destination.get_transient_table() error_type = ImportUserErrorType.query.filter_by(name=error_type).one() error_column = generated_fields.get(error_column, error_column) - error_column = imprt.fieldmapping.get(error_column, error_column) + error_column = imprt.fieldmapping.get(error_column, {}).get("column_src", error_column) if error_type.level in level_validity_mapping: assert entity is not None cte = ( diff --git a/backend/geonature/tests/imports/files/synthese/.~lock.valid_file.csv# b/backend/geonature/tests/imports/files/synthese/.~lock.valid_file.csv# new file mode 100644 index 0000000000..138f7bfcaf --- /dev/null +++ b/backend/geonature/tests/imports/files/synthese/.~lock.valid_file.csv# @@ -0,0 +1 @@ +,vbourgeois,pop-os,22.11.2024 13:23,file:///home/vbourgeois/.config/libreoffice/4; \ No newline at end of file