Skip to content

Commit

Permalink
Merge branch 'master' into MDEXP-758
Browse files Browse the repository at this point in the history
  • Loading branch information
alekGbuz authored Aug 1, 2024
2 parents fcdd812 + 0771423 commit fa6b217
Show file tree
Hide file tree
Showing 20 changed files with 397 additions and 10 deletions.
21 changes: 20 additions & 1 deletion descriptors/ModuleDescriptor-template.json
Original file line number Diff line number Diff line change
Expand Up @@ -447,6 +447,19 @@
"modulePermissions": [
"user-tenants.collection.get"
]
},
{
"methods": [
"POST"
],
"pathPattern": "/data-export/export-authority-deleted",
"permissionsRequired": [
"data-export.export-authority-deleted.post"
],
"modulePermissions": [
"inventory-storage.authorities.collection.get",
"user-tenants.collection.get"
]
}
]
},
Expand Down Expand Up @@ -631,6 +644,11 @@
"displayName": "Data Export - deleted MARC IDs",
"description": "To return deleted MARC IDs"
},
{
"permissionName": "data-export.export-authority-deleted.post",
"displayName": "Data Export - deleted authorities",
"description": "To return deleted authorities"
},
{
"permissionName": "data-export.all",
"displayName": "Data Export - all permissions",
Expand Down Expand Up @@ -661,7 +679,8 @@
"data-export.configuration.post",
"data-export.export.all",
"data-export.related-users.collection.get",
"data-export.export-deleted.post"
"data-export.export-deleted.post",
"data-export.export-authority-deleted.post"
],
"visible": false
}
Expand Down
1 change: 1 addition & 0 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
**/src/main/java/org/folio/dataexp/domain/dto/**,
**/src/main/java/org/folio/dataexp/repository**,
**/src/main/java/org/folio/dataexp/util/ExternalPathResolver.java,
**/src/main/java/org/folio/dataexp/util/Constants.java,
**/src/**/test/**/*
</sonar.exclusions>
<argLine />
Expand Down
16 changes: 16 additions & 0 deletions src/main/java/org/folio/dataexp/client/AuthorityClient.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package org.folio.dataexp.client;

import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE;

import org.folio.dataexp.domain.dto.AuthorityCollection;
import org.springframework.cloud.openfeign.FeignClient;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestParam;

@FeignClient(name = "authority-storage/authorities")
public interface AuthorityClient {

@GetMapping(produces = APPLICATION_JSON_VALUE)
AuthorityCollection getAuthorities(@RequestParam boolean idOnly, @RequestParam boolean deleted, @RequestParam String query,
@RequestParam long limit, @RequestParam long offset);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
package org.folio.dataexp.controllers;

import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.folio.dataexp.domain.dto.ExportAuthorityDeletedRequest;
import org.folio.dataexp.domain.dto.ExportAuthorityDeletedResponse;
import org.folio.dataexp.rest.resource.ExportAuthorityDeletedApi;
import org.folio.dataexp.service.ExportAuthorityDeletedService;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

@RestController
@RequiredArgsConstructor
@Log4j2
@RequestMapping("/data-export")
public class DataExportAuthorityDeletedController implements ExportAuthorityDeletedApi {

private final ExportAuthorityDeletedService exportAuthorityDeletedService;

@Override
public ResponseEntity<ExportAuthorityDeletedResponse> postExportDeletedAuthority(ExportAuthorityDeletedRequest request) {
var response = exportAuthorityDeletedService.postExportDeletedAuthority(request);
return new ResponseEntity<>(response, HttpStatus.OK);
}
}
10 changes: 10 additions & 0 deletions src/main/java/org/folio/dataexp/domain/dto/Authority.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
package org.folio.dataexp.domain.dto;

import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Data;

@Data
@JsonIgnoreProperties(ignoreUnknown = true)
public class Authority {
private String id;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package org.folio.dataexp.domain.dto;

import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data;

import java.util.ArrayList;
import java.util.List;

@Data
@JsonIgnoreProperties(ignoreUnknown = true)
public class AuthorityCollection {

@JsonProperty("authorities")
private List<Authority> authorities = new ArrayList<>();
}
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
package org.folio.dataexp.exception;

import jakarta.persistence.EntityNotFoundException;
import jakarta.validation.ConstraintViolationException;
import org.folio.dataexp.domain.dto.Errors;
import org.folio.dataexp.exception.authority.AuthorityQueryException;
import org.folio.dataexp.exception.configuration.SliceSizeValidationException;
import org.folio.dataexp.exception.export.DataExportException;
import org.folio.dataexp.exception.export.ExportDeletedDateRangeException;
Expand All @@ -20,7 +20,6 @@
import org.springframework.http.converter.HttpMessageNotReadableException;
import org.springframework.web.bind.annotation.ControllerAdvice;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.method.annotation.MethodArgumentTypeMismatchException;

@ControllerAdvice
public class DataExportExceptionHandler {
Expand Down Expand Up @@ -99,4 +98,9 @@ public ResponseEntity<String> handleInvalidDateException() {
public ResponseEntity<String> handleInvalidDateRangeException(final ExportDeletedDateRangeException e) {
return new ResponseEntity<>(e.getMessage(), HttpStatus.BAD_REQUEST);
}

@ExceptionHandler(AuthorityQueryException.class)
public ResponseEntity<String> handleAuthorityQueryException(final AuthorityQueryException e) {
return new ResponseEntity<>(e.getMessage(), HttpStatus.BAD_REQUEST);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
package org.folio.dataexp.exception.authority;

public class AuthorityQueryException extends RuntimeException {
public AuthorityQueryException(String msg) {
super(msg);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
package org.folio.dataexp.service;

import static org.folio.dataexp.util.Constants.DEFAULT_AUTHORITY_DELETED_JOB_PROFILE_ID;
import static org.folio.dataexp.util.Constants.DELETED_AUTHORITIES_FILE_NAME;

import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.folio.dataexp.client.AuthorityClient;
import org.folio.dataexp.domain.dto.Authority;
import org.folio.dataexp.domain.dto.ExportAuthorityDeletedRequest;
import org.folio.dataexp.domain.dto.ExportAuthorityDeletedResponse;
import org.folio.dataexp.domain.dto.ExportRequest;
import org.folio.dataexp.domain.dto.FileDefinition;
import org.folio.dataexp.exception.authority.AuthorityQueryException;
import org.springframework.core.io.ByteArrayResource;
import org.springframework.stereotype.Service;

import java.util.List;
import java.util.UUID;

@Log4j2
@RequiredArgsConstructor
@Service
public class ExportAuthorityDeletedService {

private final AuthorityClient authorityClient;

private final FileDefinitionsService fileDefinitionsService;
private final DataExportService dataExportService;

public ExportAuthorityDeletedResponse postExportDeletedAuthority(ExportAuthorityDeletedRequest request) {
log.info("POST export deleted authorities");
try {
var authorities = authorityClient.getAuthorities(true, true, request.getQuery(), request.getLimit(),
request.getOffset());
var fileDefinition = getFileDefinition(authorities.getAuthorities().stream().map(Authority::getId).toList());
var exportRequest = ExportRequest.builder().fileDefinitionId(fileDefinition.getId()).jobProfileId(UUID.fromString(DEFAULT_AUTHORITY_DELETED_JOB_PROFILE_ID))
.all(false).quick(false).idType(ExportRequest.IdTypeEnum.AUTHORITY).build();
dataExportService.postDataExport(exportRequest);
return ExportAuthorityDeletedResponse.builder().jobExecutionId(fileDefinition.getJobExecutionId()).build();
} catch (Exception e) {
log.error(e);
throw new AuthorityQueryException(e.getMessage());
}
}

private FileDefinition getFileDefinition(List<String> authorityIds) {
var fileContent = String.join(System.lineSeparator(), authorityIds);
var fileDefinition = new FileDefinition();
fileDefinition.setSize(authorityIds.size());
fileDefinition.setUploadFormat(FileDefinition.UploadFormatEnum.CSV);
fileDefinition.setFileName(DELETED_AUTHORITIES_FILE_NAME);
fileDefinition = fileDefinitionsService.postFileDefinition(fileDefinition);
fileDefinition = fileDefinitionsService.uploadFile(fileDefinition.getId(), new ByteArrayResource(fileContent.getBytes()));
return fileDefinition;
}
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package org.folio.dataexp.service.export.strategies;

import static java.util.stream.Collectors.toMap;
import static net.minidev.json.parser.JSONParser.DEFAULT_PERMISSIVE_MODE;
import static org.folio.dataexp.service.export.Constants.DELETED_KEY;
import static org.folio.dataexp.service.export.Constants.OUTPUT_BUFFER_SIZE;
Expand All @@ -16,11 +17,13 @@
import org.folio.dataexp.domain.dto.ExportRequest;
import org.folio.dataexp.domain.dto.MappingProfile;
import org.folio.dataexp.domain.entity.ExportIdEntity;
import org.folio.dataexp.domain.entity.InstanceEntity;
import org.folio.dataexp.domain.entity.JobExecutionExportFilesEntity;
import org.folio.dataexp.domain.entity.JobExecutionExportFilesStatus;
import org.folio.dataexp.domain.entity.MarcRecordEntity;
import org.folio.dataexp.exception.TransformationRuleException;
import org.folio.dataexp.repository.ExportIdEntityRepository;
import org.folio.dataexp.repository.InstanceEntityRepository;
import org.folio.dataexp.repository.JobProfileEntityRepository;
import org.folio.dataexp.repository.MappingProfileEntityRepository;
import org.folio.dataexp.service.JobExecutionService;
Expand Down Expand Up @@ -50,6 +53,7 @@ public abstract class AbstractExportStrategy implements ExportStrategy {
protected int exportIdsBatch;
protected String exportTmpStorage;

private InstanceEntityRepository instanceEntityRepository;
private ExportIdEntityRepository exportIdEntityRepository;
private MappingProfileEntityRepository mappingProfileEntityRepository;
private JobProfileEntityRepository jobProfileEntityRepository;
Expand All @@ -60,6 +64,7 @@ public abstract class AbstractExportStrategy implements ExportStrategy {
protected MarcAuthorityRecordAllRepository marcAuthorityRecordAllRepository;
protected FolioExecutionContext folioExecutionContext;


@PersistenceContext
protected EntityManager entityManager;

Expand Down Expand Up @@ -155,7 +160,7 @@ protected void createAndSaveMarcFromJsonRecord(Set<UUID> externalIds, ExportStra
var marc = StringUtils.EMPTY;
try {
var marcHoldingsItemsFields = additionalFieldsPerId.getOrDefault(marcRecordEntity.getExternalId(), new MarcFields());
if (marcHoldingsItemsFields.getErrorMessages().size() > 0) {
if (!marcHoldingsItemsFields.getErrorMessages().isEmpty()) {
errorLogService
.saveGeneralErrorWithMessageValues(ERROR_FIELDS_MAPPING_SRS.getCode(), marcHoldingsItemsFields.getErrorMessages(), jobExecutionId);
}
Expand Down Expand Up @@ -199,7 +204,8 @@ protected boolean isDeletedJobProfile(UUID jobProfileId) {
private void saveDuplicateErrors(LinkedHashMap<UUID, Optional<ExportIdentifiersForDuplicateErrors>> duplicatedUuidWithIdentifiers,
List<MarcRecordEntity> marcRecords, UUID jobExecutionId) {
var externalIdsAsKeys = duplicatedUuidWithIdentifiers.keySet();
var srsIdByExternalId = getSrsIdByDeletedExternalIdMap(marcRecords);
var srsIdByExternalId = getSrsIdByExternalIdMap(marcRecords);
var deletedSrsIdByExternalId = getSrsIdByDeletedExternalIdMap(marcRecords);
for (var externalId : externalIdsAsKeys) {
var exportIdentifiersOpt = duplicatedUuidWithIdentifiers.get(externalId);
if (exportIdentifiersOpt.isPresent()) {
Expand All @@ -208,13 +214,17 @@ private void saveDuplicateErrors(LinkedHashMap<UUID, Optional<ExportIdentifiersF
log.warn(errorMessage);
if (exportIdentifiers.getAssociatedJsonObject() != null) {
var associatedJson = exportIdentifiers.getAssociatedJsonObject();
if (srsIdByExternalId.containsKey(externalId)) {
if (deletedSrsIdByExternalId.containsKey(externalId)) {
associatedJson.put(DELETED_KEY, true);
errorLogService.saveGeneralErrorWithMessageValues(ErrorCode.ERROR_DELETED_DUPLICATED_INSTANCE.getCode(), List.of(srsIdByExternalId.get(externalId).toString()), jobExecutionId);
log.error(String.format(ErrorCode.ERROR_DELETED_DUPLICATED_INSTANCE.getDescription(), srsIdByExternalId.get(externalId)));
errorLogService.saveGeneralErrorWithMessageValues(ErrorCode.ERROR_DELETED_DUPLICATED_INSTANCE.getCode(), List.of(deletedSrsIdByExternalId.get(externalId).toString()), jobExecutionId);
log.error(String.format(ErrorCode.ERROR_DELETED_DUPLICATED_INSTANCE.getDescription(), deletedSrsIdByExternalId.get(externalId)));
}
errorLogService.saveWithAffectedRecord(associatedJson, errorMessage, ErrorCode.ERROR_DUPLICATE_SRS_RECORD.getCode(), jobExecutionId);
} else {
if (instanceEntityRepository.findByIdIn(Set.of(externalId)).isEmpty()) {
errorLogService.saveGeneralErrorWithMessageValues(ErrorCode.ERROR_NON_EXISTING_INSTANCE.getCode(),
List.of(String.format(ErrorCode.ERROR_NON_EXISTING_INSTANCE.getDescription(), srsIdByExternalId.get(externalId))), jobExecutionId);
}
errorLogService.saveGeneralErrorWithMessageValues(ErrorCode.ERROR_DUPLICATE_SRS_RECORD.getCode(), List.of(errorMessage), jobExecutionId);
}
}
Expand All @@ -234,7 +244,14 @@ private String getDuplicatedSRSErrorMessage(UUID externalId, List<MarcRecordEnti
}

private Map<UUID, UUID> getSrsIdByDeletedExternalIdMap(List<MarcRecordEntity> marcRecords) {
return marcRecords.stream().filter(marc -> marc.isDeleted()).collect(Collectors.toMap(marc -> marc.getExternalId(), marc -> marc.getId(), (srsId1, srsId2) -> srsId1));
return marcRecords.stream()
.filter(MarcRecordEntity::isDeleted)
.collect(toMap(MarcRecordEntity::getExternalId, MarcRecordEntity::getId, (srsId1, srsId2) -> srsId1));
}

private Map<UUID, UUID> getSrsIdByExternalIdMap(List<MarcRecordEntity> marcRecords) {
return marcRecords.stream()
.collect(toMap(MarcRecordEntity::getExternalId, MarcRecordEntity::getId, (srsId1, srsId2) -> srsId1));
}

private MappingProfile getMappingProfile(UUID jobExecutionId) {
Expand All @@ -259,6 +276,11 @@ protected void processSlices(JobExecutionExportFilesEntity exportFilesEntity,
}
}

@Autowired
private void setInstanceEntityRepository(InstanceEntityRepository instanceEntityRepository) {
this.instanceEntityRepository = instanceEntityRepository;
}

@Autowired
private void setExportIdEntityRepository(ExportIdEntityRepository exportIdEntityRepository) {
this.exportIdEntityRepository = exportIdEntityRepository;
Expand Down
2 changes: 2 additions & 0 deletions src/main/java/org/folio/dataexp/util/Constants.java
Original file line number Diff line number Diff line change
Expand Up @@ -11,5 +11,7 @@ public class Constants {
public static final String COMMA = ",";
public static final String DATE_PATTERN = "yyyyMMdd";
public static final String DEFAULT_INSTANCE_JOB_PROFILE_ID = "6f7f3cd7-9f24-42eb-ae91-91af1cd54d0a";
public static final String DEFAULT_AUTHORITY_DELETED_JOB_PROFILE_ID = "2c9be114-6d35-4408-adac-9ead35f51a27";
public static final String DELETED_MARC_IDS_FILE_NAME = "deleted-marc-bib-records.csv";
public static final String DELETED_AUTHORITIES_FILE_NAME = "deleted-authority-records.csv";
}
3 changes: 2 additions & 1 deletion src/main/java/org/folio/dataexp/util/ErrorCode.java
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,8 @@ public enum ErrorCode {
ERROR_CONVERTING_TO_JSON_HOLDING("error.convertingToJson.holding", "Error converting to json holding by id %s"),
ERROR_CONVERTING_TO_JSON_INSTANCE("error.convertingToJson.instance", "Error converting to json instance by id %s"),
ERROR_DELETED_DUPLICATED_INSTANCE("error.deletedDuplicate.instance", "Instance record associated with %s has been deleted."),
ERROR_DELETED_TOO_LONG_INSTANCE("error.deletedTooLong.instance", "Instance record with id = %s has been deleted.");
ERROR_DELETED_TOO_LONG_INSTANCE("error.deletedTooLong.instance", "Instance record with id = %s has been deleted."),
ERROR_NON_EXISTING_INSTANCE("error.nonExisting.instance", "%s");

private final String code;
private final String description;
Expand Down
41 changes: 41 additions & 0 deletions src/main/resources/swagger.api/data-export.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -836,6 +836,43 @@ paths:
content:
text/plain:
example: "Internal server error"
/export-authority-deleted:
post:
description: Export of deleted authority records
operationId: postExportDeletedAuthority
requestBody:
required: false
content:
application/json:
schema:
$ref: '#/components/schemas/exportAuthorityDeletedRequest'
example:
$ref: 'examples/exportAuthorityDeletedRequest.json'
responses:
'200':
description: Data export of deleted authority completed
content:
application/json:
schema:
$ref: '#/components/schemas/exportAuthorityDeletedResponse'
example:
$ref: 'examples/exportAuthorityDeletedResponse.json'
'400':
description: Bad request
content:
text/plain:
example: "invalid request parameters"
'422':
description: Unprocessable Entity
content:
application/json:
schema:
$ref: '#/components/schemas/errors'
'500':
description: Internal server errors, e.g. due to misconfiguration
content:
text/plain:
example: "Internal server error"
components:
schemas:
uuid:
Expand Down Expand Up @@ -973,6 +1010,10 @@ components:
$ref: 'schemas/exportDeletedMarcIdsRequest.json'
exportDeletedMarcIdsResponse:
$ref: 'schemas/exportDeletedMarcIdsResponse.json'
exportAuthorityDeletedRequest:
$ref: 'schemas/exportAuthorityDeletedRequest.json'
exportAuthorityDeletedResponse:
$ref: 'schemas/exportAuthorityDeletedResponse.json'

parameters:
trait_queryable:
Expand Down
Loading

0 comments on commit fa6b217

Please sign in to comment.