Skip to content

Commit

Permalink
MDEXP- 726 Extend mapping profile schema (#472)
Browse files Browse the repository at this point in the history 
* MDEXP-726 - mapping profile suppression

* MDEXP-726 - mapping profile suppression validation

* MDEXP-726 - mapping profile suppression validation

* MDEXP-726 - fix tests

* MDEXP-726 - fix sonar issues

* MDEXP-726 - update validation tests

* MDEXP-726 - update validation tests

* MDEXP-726 - suppression as string

* MDEXP-726 - suppression as string

* MDEXP-726 - fix sonar

* MDEXP-726 - error message
  • Loading branch information
@alekGbuz
alekGbuz authored Apr 17, 2024
1 parent ccbc8cd commit ec6706c
Show file tree
Hide file tree
Showing 9 changed files with 287 additions and 47 deletions.
12 changes: 12 additions & 0 deletions src/main/java/org/folio/dataexp/exception/DataExportExceptionHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
import org.folio.dataexp.exception.file.definition.UploadFileException;
import org.folio.dataexp.exception.job.profile.DefaultJobProfileException;
import org.folio.dataexp.exception.mapping.profile.DefaultMappingProfileException;
import org.folio.dataexp.exception.mapping.profile.MappingProfileFieldsSuppressionException;
import org.folio.dataexp.exception.mapping.profile.MappingProfileFieldsSuppressionPatternException;
import org.folio.dataexp.exception.mapping.profile.MappingProfileTransformationEmptyException;
import org.folio.dataexp.exception.mapping.profile.MappingProfileTransformationPatternException;
import org.springframework.http.HttpStatus;
Expand Down Expand Up @@ -49,6 +51,16 @@ public ResponseEntity<String> handleMappingProfileTransformationEmptyException(f
return new ResponseEntity<>(e.getMessage(), HttpStatus.UNPROCESSABLE_ENTITY);
}

@ExceptionHandler(MappingProfileFieldsSuppressionPatternException.class)
public ResponseEntity<Errors> handleMappingProfileFieldsSuppressionPatternException(final MappingProfileFieldsSuppressionPatternException e) {
return new ResponseEntity<>(e.getErrors(), HttpStatus.UNPROCESSABLE_ENTITY);
}

@ExceptionHandler(MappingProfileFieldsSuppressionException.class)
public ResponseEntity<Errors> handleMappingProfileFieldsSuppressionException(final MappingProfileFieldsSuppressionException e) {
return new ResponseEntity<>(HttpStatus.UNPROCESSABLE_ENTITY);
}

@ExceptionHandler(DefaultJobProfileException.class)
public ResponseEntity<String> handleDefaultJobProfileException(final DefaultJobProfileException e) {
return new ResponseEntity<>(e.getMessage(), HttpStatus.FORBIDDEN);
Expand Down
7 changes: 7 additions & 0 deletions ...org/folio/dataexp/exception/mapping/profile/MappingProfileFieldsSuppressionException.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package org.folio.dataexp.exception.mapping.profile;

public class MappingProfileFieldsSuppressionException extends RuntimeException {
public MappingProfileFieldsSuppressionException(String message) {
super(message);
}
}
14 changes: 14 additions & 0 deletions ...io/dataexp/exception/mapping/profile/MappingProfileFieldsSuppressionPatternException.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
package org.folio.dataexp.exception.mapping.profile;

import lombok.Getter;
import org.folio.dataexp.domain.dto.Errors;

public class MappingProfileFieldsSuppressionPatternException extends RuntimeException {
@Getter
private final Errors errors;

public MappingProfileFieldsSuppressionPatternException(String message, Errors errors) {
super(message);
this.errors = errors;
}
}
52 changes: 5 additions & 47 deletions src/main/java/org/folio/dataexp/service/MappingProfileService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,43 +3,31 @@
import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.StringUtils;
import org.folio.dataexp.client.UserClient;
import org.folio.dataexp.domain.dto.Errors;
import org.folio.dataexp.domain.dto.MappingProfile;
import org.folio.dataexp.domain.dto.MappingProfileCollection;
import org.folio.dataexp.domain.dto.Metadata;
import org.folio.dataexp.domain.dto.ParametersInner;
import org.folio.dataexp.domain.dto.RecordTypes;
import org.folio.dataexp.domain.dto.UserInfo;
import org.folio.dataexp.domain.entity.MappingProfileEntity;
import org.folio.dataexp.exception.mapping.profile.DefaultMappingProfileException;
import org.folio.dataexp.exception.mapping.profile.MappingProfileTransformationEmptyException;
import org.folio.dataexp.exception.mapping.profile.MappingProfileTransformationPatternException;
import org.folio.dataexp.repository.MappingProfileEntityCqlRepository;
import org.folio.dataexp.repository.MappingProfileEntityRepository;
import org.folio.dataexp.service.validators.MappingProfileValidator;
import org.folio.spring.FolioExecutionContext;
import org.folio.spring.data.OffsetRequest;
import org.springframework.stereotype.Service;

import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.UUID;
import java.util.regex.Pattern;

@Service
@RequiredArgsConstructor
public class MappingProfileService {

private static final Pattern TRANSFORMATION_PATTERN = Pattern.compile("((\\d{3}([\\s]|[\\d]|[a-zA-Z]){2}(\\$([a-zA-Z]|[\\d]{1,2}))?)|(^$))");
private static final String ERROR_CODE = "javax.validation.constraints.Pattern.message";
private static final String ERROR_VALIDATION_PARAMETER_KEY_PATTERN = "transformations[%s].transformation";
private static final String ERROR_VALIDATION_MESSAGE_PATTERN = "must match \\\"%s\\\"";
private static final String TRANSFORMATION_ITEM_EMPTY_VALUE_MESSAGE = "Transformations for fields with item record type cannot be empty. Please provide a value.";

private final FolioExecutionContext folioExecutionContext;
private final MappingProfileEntityRepository mappingProfileEntityRepository;
private final MappingProfileEntityCqlRepository mappingProfileEntityCqlRepository;
private final UserClient userClient;
private final MappingProfileValidator mappingProfileValidator;

public void deleteMappingProfileById(UUID mappingProfileId) {
var mappingProfileEntity = mappingProfileEntityRepository.getReferenceById(mappingProfileId);
Expand Down Expand Up @@ -82,7 +70,7 @@ public MappingProfile postMappingProfile(MappingProfile mappingProfile) {
metaData.updatedByUsername(user.getUsername());
mappingProfile.setMetadata(metaData);

validateMappingProfileTransformations(mappingProfile);
mappingProfileValidator.validate(mappingProfile);

var saved = mappingProfileEntityRepository.save(MappingProfileEntity.fromMappingProfile(mappingProfile));
return saved.getMappingProfile();
Expand Down Expand Up @@ -116,39 +104,9 @@ public void putMappingProfile(UUID mappingProfileId, MappingProfile mappingProfi

mappingProfile.setMetadata(metadata);

mappingProfileValidator.validate(mappingProfile);

mappingProfileEntityRepository.save(MappingProfileEntity.fromMappingProfile(mappingProfile));
}

private void validateMappingProfileTransformations(MappingProfile mappingProfile) {
var transformations = mappingProfile.getTransformations();
var parameters = new ArrayList<ParametersInner>();
for (int i = 0; i < transformations.size(); i++) {
var transformation = transformations.get(i);
var matcher = TRANSFORMATION_PATTERN.matcher(transformation.getTransformation());
if (!matcher.matches()) {
var parameter = ParametersInner.builder()
.key(String.format(ERROR_VALIDATION_PARAMETER_KEY_PATTERN, i))
.value(transformation.getTransformation()).build();
parameters.add(parameter);
}
}
if (!parameters.isEmpty()) {
var errors = new Errors();
for (var parameter : parameters) {
var errorItem = new org.folio.dataexp.domain.dto.Error();
errorItem.setCode(ERROR_CODE);
errorItem.type("1");
errorItem.message(String.format(ERROR_VALIDATION_MESSAGE_PATTERN, TRANSFORMATION_PATTERN));
errors.addErrorsItem(errorItem);
errorItem.setParameters(List.of(parameter));
}
errors.setTotalRecords(errors.getErrors().size());
throw new MappingProfileTransformationPatternException("Mapping profile validation exception", errors);
}
for (var transformation : transformations) {
if (StringUtils.isEmpty(transformation.getTransformation()) && transformation.getRecordType() == RecordTypes.ITEM) {
throw new MappingProfileTransformationEmptyException(TRANSFORMATION_ITEM_EMPTY_VALUE_MESSAGE);
}
}
}
}
109 changes: 109 additions & 0 deletions src/main/java/org/folio/dataexp/service/validators/MappingProfileValidator.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
package org.folio.dataexp.service.validators;

import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.StringUtils;
import org.folio.dataexp.domain.dto.Errors;
import org.folio.dataexp.domain.dto.MappingProfile;
import org.folio.dataexp.domain.dto.ParametersInner;
import org.folio.dataexp.domain.dto.RecordTypes;
import org.folio.dataexp.exception.mapping.profile.MappingProfileFieldsSuppressionException;
import org.folio.dataexp.exception.mapping.profile.MappingProfileFieldsSuppressionPatternException;
import org.folio.dataexp.exception.mapping.profile.MappingProfileTransformationEmptyException;
import org.folio.dataexp.exception.mapping.profile.MappingProfileTransformationPatternException;
import org.springframework.stereotype.Component;

import java.util.ArrayList;
import java.util.List;
import java.util.regex.Pattern;

@Component
@RequiredArgsConstructor
public class MappingProfileValidator {

private static final String VALIDATION_ERROR_MESSAGE = "Mapping profile validation exception";
private static final String ERROR_CODE = "javax.validation.constraints.Pattern.message";
private static final String ERROR_VALIDATION_MESSAGE_PATTERN = "must match \\\"%s\\\"";

private static final Pattern TRANSFORMATION_PATTERN = Pattern.compile("((\\d{3}([\\s]|[\\d]|[a-zA-Z]){2}(\\$([a-zA-Z]|[\\d]{1,2}))?)|(^$))");
private static final String ERROR_VALIDATION_TRANSFORMATIONS_PARAMETER_KEY_PATTERN = "transformations[%s].transformation";
private static final String TRANSFORMATION_ITEM_EMPTY_VALUE_MESSAGE = "Transformations for fields with item record type cannot be empty. Please provide a value.";

private static final Pattern SUPPRESSION_FIELD_PATTERN = Pattern.compile("^\\d{3}$");
private static final String ERROR_VALIDATION_SUPPRESSION_FIELD_PARAMETER_KEY_PATTERN = "suppressionFields[%s]";
private static final String ERROR_USAGE_SUPPRESSION_FIELD_FOR_ITEM_RECORD_TYPE = "Suppression field can not be used only for Item record type";
private static final String SUPPRESSION_VALIDATION_ERROR_MESSAGE = "Suppressed fields can be represented by three digits only and need to be separated by a comma.";

public void validate(MappingProfile mappingProfile) {
validateMappingProfileTransformations(mappingProfile);
validateMappingProfileSuppression(mappingProfile);
}

private void validateMappingProfileTransformations(MappingProfile mappingProfile) {
var transformations = mappingProfile.getTransformations();
var parameters = new ArrayList<ParametersInner>();
for (int i = 0; i < transformations.size(); i++) {
var transformation = transformations.get(i);
var matcher = TRANSFORMATION_PATTERN.matcher(transformation.getTransformation());
if (!matcher.matches()) {
var parameter = ParametersInner.builder()
.key(String.format(ERROR_VALIDATION_TRANSFORMATIONS_PARAMETER_KEY_PATTERN, i))
.value(transformation.getTransformation()).build();
parameters.add(parameter);
}
}
if (!parameters.isEmpty()) {
var errors = new Errors();
for (var parameter : parameters) {
var errorItem = new org.folio.dataexp.domain.dto.Error();
errorItem.setCode(ERROR_CODE);
errorItem.type("1");
errorItem.message(String.format(ERROR_VALIDATION_MESSAGE_PATTERN, TRANSFORMATION_PATTERN));
errors.addErrorsItem(errorItem);
errorItem.setParameters(List.of(parameter));
}
errors.setTotalRecords(errors.getErrors().size());
throw new MappingProfileTransformationPatternException(VALIDATION_ERROR_MESSAGE, errors);
}
for (var transformation : transformations) {
if (StringUtils.isEmpty(transformation.getTransformation()) && transformation.getRecordType() == RecordTypes.ITEM) {
throw new MappingProfileTransformationEmptyException(TRANSFORMATION_ITEM_EMPTY_VALUE_MESSAGE);
}
}
}

private void validateMappingProfileSuppression(MappingProfile mappingProfile) {
var recordTypes = mappingProfile.getRecordTypes();
boolean isExistAllItemRecordType = recordTypes.stream().allMatch(type -> type == RecordTypes.ITEM);
var fieldsSuppressionAsStr = mappingProfile.getFieldsSuppression();
if (StringUtils.isNotEmpty(fieldsSuppressionAsStr)) {
if (!recordTypes.isEmpty() && isExistAllItemRecordType) {
throw new MappingProfileFieldsSuppressionException(ERROR_USAGE_SUPPRESSION_FIELD_FOR_ITEM_RECORD_TYPE);
}
var fieldsSuppression = fieldsSuppressionAsStr.split(",");
var parameters = new ArrayList<ParametersInner>();
for (int i = 0; i < fieldsSuppression.length; i++) {
var suppression = StringUtils.trim(fieldsSuppression[i]);
var matcher = SUPPRESSION_FIELD_PATTERN.matcher(suppression);
if (!matcher.matches()) {
var parameter = ParametersInner.builder()
.key(String.format(ERROR_VALIDATION_SUPPRESSION_FIELD_PARAMETER_KEY_PATTERN, i))
.value(suppression).build();
parameters.add(parameter);
}
}
if (!parameters.isEmpty()) {
var errors = new Errors();
for (var parameter : parameters) {
var errorItem = new org.folio.dataexp.domain.dto.Error();
errorItem.setCode(ERROR_CODE);
errorItem.type("1");
errorItem.message(String.format(ERROR_VALIDATION_MESSAGE_PATTERN, SUPPRESSION_FIELD_PATTERN));
errors.addErrorsItem(errorItem);
errorItem.setParameters(List.of(parameter));
}
errors.setTotalRecords(errors.getErrors().size());
throw new MappingProfileFieldsSuppressionPatternException(SUPPRESSION_VALIDATION_ERROR_MESSAGE, errors);
}
}
}
}
9 changes: 9 additions & 0 deletions src/main/resources/swagger.api/schemas/profiles/mappingProfile.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,15 @@
"MARC"
]
},
"fieldsSuppression": {
"description": "Fields to suppress",
"type": "string"
},
"suppress999ff": {
"description": "Supress 999 field rule",
"type": "boolean",
"default": false
},
"metadata": {
"description": "Metadata provided by the server",
"type": "object",
Expand Down
54 changes: 54 additions & 0 deletions src/test/java/org/folio/dataexp/controllers/MappingProfileControllerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,7 @@ void postMappingProfileTest() {
mappingProfile.setDefault(true);
mappingProfile.setName("mappingProfile");
mappingProfile.setTransformations(List.of(transformation));
mappingProfile.setFieldsSuppression("902");
var user = new User();
user.setPersonal(new User.Personal());

Expand Down Expand Up @@ -272,13 +273,60 @@ void postMappingProfileTestIfItemTransformationEmptyAndTransformationsNotMatchPa
assertEquals("902q $bbb", error.getParameters().get(0).getValue());
}

@Test
@SneakyThrows
void postMappingProfileIfSuppressionNotMatchTest() {
var transformation = new Transformations();
transformation.setFieldId("holdings.callnumber");
transformation.setPath("$.holdings[*].callNumber");
transformation.setRecordType(RecordTypes.HOLDINGS);
transformation.setTransformation("900 $a");
var mappingProfile = new MappingProfile();
mappingProfile.setId(UUID.randomUUID());
mappingProfile.setDefault(true);
mappingProfile.setName("mappingProfile");
mappingProfile.setTransformations(List.of(transformation));
mappingProfile.setFieldsSuppression("897 , 90");
var user = new User();
user.setPersonal(new User.Personal());

var entity = MappingProfileEntity.builder().id(mappingProfile.getId()).mappingProfile(mappingProfile).build();
when(mappingProfileEntityRepository.save(isA(MappingProfileEntity.class))).thenReturn(entity);
when(userClient.getUserById(isA(String.class))).thenReturn(user);

var result = mockMvc.perform(MockMvcRequestBuilders
.post("/data-export/mapping-profiles")
.headers(defaultHeaders())
.content(asJsonString(mappingProfile)))
.andExpect(status().isUnprocessableEntity()).andReturn();

var response = result.getResponse().getContentAsString();
var mapper = new ObjectMapper();

var errors = mapper.readValue(response, Errors.class);
assertEquals(1, errors.getErrors().size());

var error = errors.getErrors().get(0);

assertEquals("must match \\\"^\\d{3}$\\\"", error.getMessage());
assertEquals(1, error.getParameters().size());
assertEquals("suppressionFields[1]", error.getParameters().get(0).getKey());
assertEquals("90", error.getParameters().get(0).getValue());
}

@Test
@SneakyThrows
void putMappingProfileTest() {
var transformation = new Transformations();
transformation.setFieldId("holdings.callnumber");
transformation.setPath("$.holdings[*].callNumber");
transformation.setRecordType(RecordTypes.HOLDINGS);
transformation.setTransformation("900 $a");
var mappingProfile = new MappingProfile();
mappingProfile.setId(UUID.randomUUID());
mappingProfile.setDefault(false);
mappingProfile.setName("mappingProfile");
mappingProfile.setTransformations(List.of(transformation));
mappingProfile.setMetadata(new Metadata().createdDate(new Date()));
var user = new User();
user.setPersonal(new User.Personal());
Expand All @@ -300,10 +348,16 @@ void putMappingProfileTest() {
@Test
@SneakyThrows
void putDefaultMappingProfileTest() {
var transformation = new Transformations();
transformation.setFieldId("holdings.callnumber");
transformation.setPath("$.holdings[*].callNumber");
transformation.setRecordType(RecordTypes.HOLDINGS);
transformation.setTransformation("900 $a");
var mappingProfile = new MappingProfile();
mappingProfile.setId(UUID.randomUUID());
mappingProfile.setDefault(true);
mappingProfile.setName("mappingProfile");
mappingProfile.setTransformations(List.of(transformation));

var entity = MappingProfileEntity.builder().id(mappingProfile.getId()).mappingProfile(mappingProfile).build();
when(mappingProfileEntityRepository.getReferenceById(isA(UUID.class))).thenReturn(entity);
Expand Down
5 changes: 5 additions & 0 deletions src/test/java/org/folio/dataexp/service/MappingProfileServiceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import org.folio.dataexp.domain.entity.MappingProfileEntity;
import org.folio.dataexp.repository.MappingProfileEntityCqlRepository;
import org.folio.dataexp.repository.MappingProfileEntityRepository;
import org.folio.dataexp.service.validators.MappingProfileValidator;
import org.folio.spring.FolioExecutionContext;
import org.folio.spring.data.OffsetRequest;
import org.junit.jupiter.api.Test;
Expand All @@ -36,6 +37,8 @@ class MappingProfileServiceTest {
@Mock
private MappingProfileEntityCqlRepository mappingProfileEntityCqlRepository;
@Mock
private MappingProfileValidator mappingProfileValidator;
@Mock
private UserClient userClient;

@InjectMocks
Expand Down Expand Up @@ -114,6 +117,7 @@ void postMappingProfileTest() {
mappingProfileService.postMappingProfile(mappingProfile);

verify(mappingProfileEntityRepository).save(isA(MappingProfileEntity.class));
verify(mappingProfileValidator).validate(isA(MappingProfile.class));
}

@Test
Expand All @@ -135,5 +139,6 @@ void putMappingProfileTest() {
mappingProfileService.putMappingProfile(mappingProfile.getId(), mappingProfile);

verify(mappingProfileEntityRepository).save(isA(MappingProfileEntity.class));
verify(mappingProfileValidator).validate(isA(MappingProfile.class));
}
}
Loading

0 comments on commit ec6706c

Please sign in to comment.