Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/MDEXP-683' into MDEXP-683
Browse files Browse the repository at this point in the history
  • Loading branch information
@obozhko-folio
obozhko-folio committed Apr 29, 2024
2 parents 47845c2 + 63da3d6 commit b172ba9
Show file tree
Hide file tree
Showing 9 changed files with 160 additions and 21 deletions.
2 changes: 2 additions & 0 deletions src/main/java/org/folio/dataexp/exception/DataExportExceptionHandler.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package org.folio.dataexp.exception;

import jakarta.persistence.EntityNotFoundException;
import jakarta.validation.ConstraintViolationException;
import org.folio.dataexp.domain.dto.Errors;
import org.folio.dataexp.exception.configuration.SliceSizeValidationException;
import org.folio.dataexp.exception.export.DataExportException;
Expand All @@ -19,6 +20,7 @@
import org.springframework.http.converter.HttpMessageNotReadableException;
import org.springframework.web.bind.annotation.ControllerAdvice;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.method.annotation.MethodArgumentTypeMismatchException;

@ControllerAdvice
public class DataExportExceptionHandler {
Expand Down
26 changes: 15 additions & 11 deletions src/main/java/org/folio/dataexp/service/export/strategies/AbstractExportStrategy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,13 +129,13 @@ protected void createAndSaveMarc(Set<UUID> externalIds, ExportStrategyStatistic
UUID jobExecutionId, ExportRequest exportRequest, LocalStorageWriter localStorageWriter) {
var externalIdsWithMarcRecord = new HashSet<UUID>();
var marcRecords = getMarcRecords(externalIds, mappingProfile, exportRequest, jobExecutionId);
createMarc(externalIds, exportStatistic, mappingProfile, jobExecutionId, externalIdsWithMarcRecord, marcRecords, localStorageWriter);
createAndSaveMarcFromJsonRecord(externalIds, exportStatistic, mappingProfile, jobExecutionId, externalIdsWithMarcRecord, marcRecords, localStorageWriter);
var result = getGeneratedMarc(externalIds, mappingProfile, exportRequest, jobExecutionId, exportStatistic);
saveMarc(result, exportStatistic, localStorageWriter);
createAndSaveGeneratedMarc(result, exportStatistic, localStorageWriter);
}

protected void createMarc(Set<UUID> externalIds, ExportStrategyStatistic exportStatistic, MappingProfile mappingProfile,
UUID jobExecutionId, Set<UUID> externalIdsWithMarcRecord, List<MarcRecordEntity> marcRecords, LocalStorageWriter localStorageWriter) {
protected void createAndSaveMarcFromJsonRecord(Set<UUID> externalIds, ExportStrategyStatistic exportStatistic, MappingProfile mappingProfile,
UUID jobExecutionId, Set<UUID> externalIdsWithMarcRecord, List<MarcRecordEntity> marcRecords, LocalStorageWriter localStorageWriter) {
marcRecords = new ArrayList<>(marcRecords);
log.info("marcRecords size: {}", marcRecords.size());
Map<UUID, MarcFields> additionalFieldsPerId = null;
Expand All @@ -151,16 +151,14 @@ protected void createMarc(Set<UUID> externalIds, ExportStrategyStatistic exportS
var marc = StringUtils.EMPTY;
try {
var marcHoldingsItemsFields = additionalFieldsPerId.getOrDefault(marcRecordEntity.getExternalId(), new MarcFields());
marc = jsonToMarcConverter.convertJsonRecordToMarcRecord(marcRecordEntity.getContent(), marcHoldingsItemsFields.getHoldingItemsFields(), mappingProfile);
if (marcHoldingsItemsFields.getErrorMessages().size() > 0) {
errorLogService
.saveGeneralErrorWithMessageValues(ERROR_FIELDS_MAPPING_SRS.getCode(), marcHoldingsItemsFields.getErrorMessages(), jobExecutionId);
.saveGeneralErrorWithMessageValues(ERROR_FIELDS_MAPPING_SRS.getCode(), marcHoldingsItemsFields.getErrorMessages(), jobExecutionId);
}
marc = jsonToMarcConverter.convertJsonRecordToMarcRecord(marcRecordEntity.getContent(), marcHoldingsItemsFields.getHoldingItemsFields(), mappingProfile);
} catch (Exception e) {
var errorMessage = "Error converting json to marc for record " + marcRecordEntity.getExternalId().toString();
log.error(errorMessage);
exportStatistic.incrementFailed();
errorLogService.saveGeneralError(errorMessage, jobExecutionId);
saveConvertJsonRecordToMarcRecordError(marcRecordEntity, jobExecutionId, e);
continue;
}
localStorageWriter.write(marc);
Expand All @@ -178,7 +176,7 @@ protected void createMarc(Set<UUID> externalIds, ExportStrategyStatistic exportS
externalIds.removeAll(externalIdsWithMarcRecord);
}

protected void saveMarc(GeneratedMarcResult result, ExportStrategyStatistic exportStatistic, LocalStorageWriter localStorageWriter) {
protected void createAndSaveGeneratedMarc(GeneratedMarcResult result, ExportStrategyStatistic exportStatistic, LocalStorageWriter localStorageWriter) {
log.info("Generated marc size: {}", result.getMarcRecords().size());
result.getMarcRecords().forEach(marc -> {
if (StringUtils.isNotEmpty(marc)) {
Expand Down Expand Up @@ -212,6 +210,12 @@ private void saveDuplicateErrors(LinkedHashMap<UUID, Optional<ExportIdentifiersF
}
}

public void saveConvertJsonRecordToMarcRecordError(MarcRecordEntity marcRecordEntity, UUID jobExecutionId, Exception e) {
var errorMessage = "Error converting json to marc for record " + marcRecordEntity.getExternalId().toString();
log.error(errorMessage + " : " + e.getMessage());
errorLogService.saveGeneralError(errorMessage, jobExecutionId);
}

private String getDuplicatedSRSErrorMessage(UUID externalId, List<MarcRecordEntity> marcRecords, ExportIdentifiersForDuplicateErrors exportIdentifiers) {
var marcRecordIds = marcRecords.stream().filter(m -> m.getExternalId().equals(externalId))
.map(e -> e.getId().toString()).collect(Collectors.joining(", "));
Expand Down Expand Up @@ -266,7 +270,7 @@ private void setJobExecutionService(JobExecutionService jobExecutionService) {
}

@Autowired
private void setErrorLogService(ErrorLogService errorLogService) {
protected void setErrorLogService(ErrorLogService errorLogService) {
this.errorLogService = errorLogService;
}

Expand Down
2 changes: 1 addition & 1 deletion src/main/java/org/folio/dataexp/service/export/strategies/AuthorityExportAllStrategy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,6 @@ private Slice<MarcRecordEntity> chooseSlice(JobExecutionExportFilesEntity export
protected void createAndSaveMarc(Set<UUID> externalIds, List<MarcRecordEntity> marcRecords, ExportStrategyStatistic exportStatistic,
MappingProfile mappingProfile, UUID jobExecutionId, LocalStorageWriter localStorageWriter) {
var externalIdsWithMarcRecord = new HashSet<UUID>();
createMarc(externalIds, exportStatistic, mappingProfile, jobExecutionId, externalIdsWithMarcRecord, marcRecords, localStorageWriter);
createAndSaveMarcFromJsonRecord(externalIds, exportStatistic, mappingProfile, jobExecutionId, externalIdsWithMarcRecord, marcRecords, localStorageWriter);
}
}
4 changes: 2 additions & 2 deletions src/main/java/org/folio/dataexp/service/export/strategies/HoldingsExportAllStrategy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,14 +132,14 @@ private void processMarcHoldingsSlices(JobExecutionExportFilesEntity exportFiles
private void processMarcHoldings(JobExecutionExportFilesEntity exportFilesEntity, ExportStrategyStatistic exportStatistic, MappingProfile mappingProfile,
List<MarcRecordEntity> marcRecords, LocalStorageWriter localStorageWriter) {
var externalIds = marcRecords.stream().map(MarcRecordEntity::getExternalId).collect(Collectors.toSet());
createMarc(externalIds, exportStatistic, mappingProfile, exportFilesEntity.getJobExecutionId(), new HashSet<>(),
createAndSaveMarcFromJsonRecord(externalIds, exportStatistic, mappingProfile, exportFilesEntity.getJobExecutionId(), new HashSet<>(),
marcRecords, localStorageWriter);
}

private void processFolioHoldings(JobExecutionExportFilesEntity exportFilesEntity, ExportStrategyStatistic exportStatistic, MappingProfile mappingProfile,
List<HoldingsRecordEntity> folioHoldings, LocalStorageWriter localStorageWriter) {
var result = getGeneratedMarc(folioHoldings, mappingProfile, exportFilesEntity.getJobExecutionId());
saveMarc(result, exportStatistic, localStorageWriter);
createAndSaveGeneratedMarc(result, exportStatistic, localStorageWriter);
}

private Slice<HoldingsRecordEntity> nextFolioSlice(JobExecutionExportFilesEntity exportFilesEntity, ExportRequest exportRequest, Pageable pageble) {
Expand Down
27 changes: 25 additions & 2 deletions src/main/java/org/folio/dataexp/service/export/strategies/InstancesExportAllStrategy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.folio.dataexp.service.export.strategies.handlers.RuleHandler;
import org.folio.dataexp.service.logs.ErrorLogService;
import org.folio.dataexp.service.transformationfields.ReferenceDataProvider;
import org.folio.dataexp.util.ErrorCode;
import org.folio.processor.RuleProcessor;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
Expand Down Expand Up @@ -113,6 +114,28 @@ public Optional<ExportIdentifiersForDuplicateErrors> getIdentifiers(UUID id) {
return identifiers;
}

@Override
public void saveConvertJsonRecordToMarcRecordError(MarcRecordEntity marcRecordEntity, UUID jobExecutionId, Exception e) {
var instances = instanceEntityRepository.findByIdIn(Set.of(marcRecordEntity.getExternalId()));
var errorMessage = e.getMessage();
if (!instances.isEmpty() || !errorMessage.contains(LONG_MARC_RECORD_MESSAGE)) {
super.saveConvertJsonRecordToMarcRecordError(marcRecordEntity, jobExecutionId, e);
} else {
var auditInstances = auditInstanceEntityRepository.findByIdIn(Set.of(marcRecordEntity.getExternalId()));
if (!auditInstances.isEmpty()) {
var auditInstance = auditInstances.get(0);
var instanceAssociatedJsonObject = new JSONObject();
instanceAssociatedJsonObject.put(ErrorLogService.ID, auditInstance.getId());
instanceAssociatedJsonObject.put(ErrorLogService.HRID, auditInstance.getHrid());
instanceAssociatedJsonObject.put(ErrorLogService.TITLE, auditInstance.getTitle());
errorLogService.saveWithAffectedRecord(instanceAssociatedJsonObject, e.getMessage(), ErrorCode.ERROR_MESSAGE_JSON_CANNOT_BE_CONVERTED_TO_MARC.getCode(), jobExecutionId);
log.error("Error converting record to marc " + marcRecordEntity.getExternalId() + " : " + e.getMessage());
} else {
super.saveConvertJsonRecordToMarcRecordError(marcRecordEntity, jobExecutionId, e);
}
}
}

private void handleDeleted(JobExecutionExportFilesEntity exportFilesEntity, ExportStrategyStatistic exportStatistic, MappingProfile mappingProfile,
ExportRequest exportRequest, LocalStorageWriter localStorageWriter) {
var deletedFolioInstances = getFolioDeleted(exportRequest);
Expand Down Expand Up @@ -171,14 +194,14 @@ private void processMarcInstances(JobExecutionExportFilesEntity exportFilesEntit
List<MarcRecordEntity> marcRecords, LocalStorageWriter localStorageWriter) {
var externalIds = marcRecords.stream().map(MarcRecordEntity::getExternalId).collect(Collectors.toSet());
log.info("processMarcInstances instances all externalIds: {}", externalIds.size());
createMarc(externalIds, exportStatistic, mappingProfile, exportFilesEntity.getJobExecutionId(), new HashSet<>(),
createAndSaveMarcFromJsonRecord(externalIds, exportStatistic, mappingProfile, exportFilesEntity.getJobExecutionId(), new HashSet<>(),
marcRecords, localStorageWriter);
}

private void processFolioInstances(JobExecutionExportFilesEntity exportFilesEntity, ExportStrategyStatistic exportStatistic, MappingProfile mappingProfile,
List<InstanceEntity> folioInstances, LocalStorageWriter localStorageWriter) {
var result = getGeneratedMarc(folioInstances, mappingProfile, exportFilesEntity.getJobExecutionId());
saveMarc(result, exportStatistic, localStorageWriter);
createAndSaveGeneratedMarc(result, exportStatistic, localStorageWriter);
}

private Slice<InstanceEntity> nextFolioSlice(JobExecutionExportFilesEntity exportFilesEntity, ExportRequest exportRequest, Pageable pageble) {
Expand Down
21 changes: 18 additions & 3 deletions src/main/java/org/folio/dataexp/service/export/strategies/InstancesExportStrategy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@
public class InstancesExportStrategy extends AbstractExportStrategy {

protected static final String INSTANCE_MARC_TYPE = "MARC_BIB";

protected static final String LONG_MARC_RECORD_MESSAGE = "Record is too long to be a valid MARC binary record";
private final ConsortiaService consortiaService;
private final InstanceCentralTenantRepository instanceCentralTenantRepository;
private final MarcInstanceRecordRepository marcInstanceRecordRepository;
Expand Down Expand Up @@ -161,13 +161,29 @@ public Optional<ExportIdentifiersForDuplicateErrors> getIdentifiers(UUID id) {
return getDefaultIdentifiers(id);
}

@Override
public void saveConvertJsonRecordToMarcRecordError(MarcRecordEntity marcRecordEntity, UUID jobExecutionId, Exception e) {
var errorMessage = e.getMessage();
var instances = instanceEntityRepository.findByIdIn(Set.of(marcRecordEntity.getExternalId()));
if (errorMessage.contains(LONG_MARC_RECORD_MESSAGE) && !instances.isEmpty()) {
var jsonObject= getAsJsonObject(instances.get(0).getJsonb());
if (jsonObject.isPresent()) {
var instanceJson= jsonObject.get();
errorLogService.saveWithAffectedRecord(instanceJson, e.getMessage(), ErrorCode.ERROR_MESSAGE_JSON_CANNOT_BE_CONVERTED_TO_MARC.getCode(), jobExecutionId);
log.error("Error converting record to marc " + marcRecordEntity.getExternalId() + " : " + e.getMessage());
return;
}
}
super.saveConvertJsonRecordToMarcRecordError(marcRecordEntity, jobExecutionId, e);
}

protected Optional<ExportIdentifiersForDuplicateErrors> getDefaultIdentifiers(UUID id) {
var exportIdentifiers = new ExportIdentifiersForDuplicateErrors();
exportIdentifiers.setIdentifierHridMessage("Instance with ID : " + id);
return Optional.of(exportIdentifiers);
}

protected List<Rule> getRules(MappingProfile mappingProfile) throws TransformationRuleException {
private List<Rule> getRules(MappingProfile mappingProfile) throws TransformationRuleException {
List<Rule> rules;
if (mappingProfile.getRecordTypes().contains(RecordTypes.SRS)) {
var defaultMappingProfile = mappingProfileEntityRepository.getReferenceById(UUID.fromString(DEFAULT_INSTANCE_MAPPING_PROFILE_ID)).getMappingProfile();
Expand All @@ -187,7 +203,6 @@ protected List<Rule> getRules(MappingProfile mappingProfile) throws Transformati
} else {
rules = ruleFactory.getRules(mappingProfile);
}

return rules;
}

Expand Down
3 changes: 1 addition & 2 deletions src/test/java/org/folio/dataexp/service/MarcDeletedIdsServiceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.core.io.Resource;

import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.List;
Expand All @@ -29,7 +28,7 @@

@ExtendWith(MockitoExtension.class)
class MarcDeletedIdsServiceTest {

private final DateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd");

@Mock
Expand Down
63 changes: 63 additions & 0 deletions ...test/java/org/folio/dataexp/service/export/strategies/InstancesExportAllStrategyTest.java
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,29 @@
package org.folio.dataexp.service.export.strategies;

import net.minidev.json.JSONObject;
import org.folio.dataexp.domain.entity.AuditInstanceEntity;
import org.folio.dataexp.domain.entity.InstanceEntity;
import org.folio.dataexp.domain.entity.MarcRecordEntity;
import org.folio.dataexp.repository.AuditInstanceEntityRepository;
import org.folio.dataexp.repository.InstanceEntityRepository;
import org.folio.dataexp.service.logs.ErrorLogService;
import org.folio.dataexp.util.ErrorCode;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;

import java.io.IOException;
import java.util.List;
import java.util.UUID;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anySet;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.isA;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

@ExtendWith(MockitoExtension.class)
Expand All @@ -24,6 +33,8 @@ class InstancesExportAllStrategyTest {
private AuditInstanceEntityRepository auditInstanceEntityRepository;
@Mock
private InstanceEntityRepository instanceEntityRepository;
@Mock
private ErrorLogService errorLogService;

@InjectMocks
private InstancesExportAllStrategy instancesExportAllStrategy;
Expand Down Expand Up @@ -57,4 +68,56 @@ void getIdentifierMessageIfInstanceDoesNotExistTest() {
assertTrue(opt.isPresent());
assertEquals("Instance with ID : b9d26945-9757-4855-ae6e-fd5d2f7d778e", opt.get().getIdentifierHridMessage());
}

@Test
void saveConvertJsonRecordToMarcRecordErrorIfErrorRecordTooLongAndInstanceDeletedTest() {
instancesExportAllStrategy.setErrorLogService(errorLogService);

var auditInstanceEntity = AuditInstanceEntity.builder()
.id(UUID.randomUUID()).hrid("123").title("title").build();
var jobExecutionId = UUID.randomUUID();
var instanceId = UUID.fromString("1eaa1eef-1633-4c7e-af09-796315ebc576");
var marcRecord = MarcRecordEntity.builder().externalId(instanceId).build();
var errorMessage = "Record is too long to be a valid MARC binary record, it's length would be 113937 which is more thatn 99999 bytes 2024";

when(instanceEntityRepository.findByIdIn(anySet())).thenReturn(List.of());
when(auditInstanceEntityRepository.findByIdIn(anySet())).thenReturn(List.of(auditInstanceEntity));

instancesExportAllStrategy.saveConvertJsonRecordToMarcRecordError(marcRecord, jobExecutionId, new IOException(errorMessage));
verify(errorLogService).saveWithAffectedRecord(isA(JSONObject.class), eq(errorMessage), eq(ErrorCode.ERROR_MESSAGE_JSON_CANNOT_BE_CONVERTED_TO_MARC.getCode()), isA(UUID.class));
}

@Test
void saveConvertJsonRecordToMarcRecordErrorIfErrorRecordTooLongAndInstanceNotDeletedTest() {
instancesExportAllStrategy.setErrorLogService(errorLogService);

var jobExecutionId = UUID.randomUUID();
var instance = "{'id' : '1eaa1eef-1633-4c7e-af09-796315ebc576', 'hrid' : 'instHrid', 'title' : 'title'}";
var instanceId = UUID.fromString("1eaa1eef-1633-4c7e-af09-796315ebc576");
var instanceEntity = InstanceEntity.builder().jsonb(instance).id(instanceId).build();
var marcRecord = MarcRecordEntity.builder().externalId(instanceId).build();
var errorMessage = "Record is too long to be a valid MARC binary record, it's length would be 113937 which is more thatn 99999 bytes 2024";

when(instanceEntityRepository.findByIdIn(anySet())).thenReturn(List.of(instanceEntity));

instancesExportAllStrategy.saveConvertJsonRecordToMarcRecordError(marcRecord, jobExecutionId, new IOException(errorMessage));

verify(errorLogService).saveWithAffectedRecord(isA(JSONObject.class), eq(errorMessage), eq(ErrorCode.ERROR_MESSAGE_JSON_CANNOT_BE_CONVERTED_TO_MARC.getCode()), isA(UUID.class));
}


@Test
void saveConvertJsonRecordToMarcRecordErrorIfNotErrorRecordTooLongTest() {
instancesExportAllStrategy.setErrorLogService(errorLogService);

var jobExecutionId = UUID.randomUUID();
var instanceId = UUID.fromString("1eaa1eef-1633-4c7e-af09-796315ebc576");
var marcRecord = MarcRecordEntity.builder().externalId(instanceId).build();
var errorMessage = "error message";

instancesExportAllStrategy.saveConvertJsonRecordToMarcRecordError(marcRecord, jobExecutionId, new IOException(errorMessage));

var expectedErrorMessage = "Error converting json to marc for record 1eaa1eef-1633-4c7e-af09-796315ebc576";
verify(errorLogService).saveGeneralError(expectedErrorMessage, jobExecutionId);
}
}
Loading

0 comments on commit b172ba9

Please sign in to comment.