Skip to content

Commit

Permalink
Merge branch 'mdexp-spring' into MDEXP-686
Browse files Browse the repository at this point in the history
  • Loading branch information
siarhei-charniak committed Feb 20, 2024
2 parents 7148506 + dea6691 commit 7261438
Show file tree
Hide file tree
Showing 25 changed files with 457 additions and 248 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
package org.folio.dataexp.controllers;

import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.folio.dataexp.domain.dto.QuickExportRequest;
import org.folio.dataexp.domain.dto.QuickExportResponse;
import org.folio.dataexp.rest.resource.QuickExportApi;
import org.folio.dataexp.service.QuickExportService;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

@RestController
@RequiredArgsConstructor
@Log4j2
@RequestMapping("/data-export")
public class QuickExportController implements QuickExportApi {

private final QuickExportService quickExportService;

@Override
public ResponseEntity<QuickExportResponse> postDataExportQuickExport(QuickExportRequest quickExportRequest) {
var response = quickExportService.postQuickExport(quickExportRequest);
log.info("Quick export response: {}", response);
return new ResponseEntity<>(response, HttpStatus.OK);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -50,23 +50,24 @@ public class JobExecutionEntity {
@Enumerated(EnumType.STRING)
private JobExecution.StatusEnum status;

public static JobExecutionEntity from(JobExecution jobExecution) {
public static JobExecutionEntity fromJobExecution(JobExecution jobExecution) {
if (isNull(jobExecution.getId())) {
jobExecution.setId(UUID.randomUUID());
}
return JobExecutionEntity.builder()
.id(jobExecution.getId())
.jobExecution(jobExecution)
.hrid(jobExecution.getHrId())
.total(jobExecution.getProgress().getTotal())
.exported(jobExecution.getProgress().getExported())
.failed(jobExecution.getProgress().getFailed())
.total(isNull(jobExecution.getProgress()) ? null : jobExecution.getProgress().getTotal())
.exported(isNull(jobExecution.getProgress()) ? null : jobExecution.getProgress().getExported())
.failed(isNull(jobExecution.getProgress()) ? null : jobExecution.getProgress().getFailed())
.jobProfileId(jobExecution.getJobProfileId())
.jobProfileName(jobExecution.getJobProfileName())
.startedDate(jobExecution.getStartedDate())
.completedDate(jobExecution.getCompletedDate())
.runByFirstName(jobExecution.getRunBy().getFirstName())
.runByLastName(jobExecution.getRunBy().getLastName())
.runByFirstName(isNull(jobExecution.getRunBy()) ? null : jobExecution.getRunBy().getFirstName())
.runByLastName(isNull(jobExecution.getRunBy()) ? null : jobExecution.getRunBy().getLastName())
.status(jobExecution.getStatus())
.build();
}
}
30 changes: 12 additions & 18 deletions src/main/java/org/folio/dataexp/service/DataExportService.java
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,9 @@
import org.folio.dataexp.domain.dto.JobExecution;
import org.folio.dataexp.domain.dto.JobExecutionProgress;
import org.folio.dataexp.domain.dto.JobExecutionRunBy;
import org.folio.dataexp.domain.entity.JobExecutionEntity;
import org.folio.dataexp.exception.export.DataExportRequestValidationException;
import org.folio.dataexp.repository.ExportIdEntityRepository;
import org.folio.dataexp.repository.FileDefinitionEntityRepository;
import org.folio.dataexp.repository.JobExecutionEntityRepository;
import org.folio.dataexp.repository.JobProfileEntityRepository;
import org.folio.dataexp.service.validators.DataExportRequestValidator;
import org.folio.spring.FolioExecutionContext;
Expand All @@ -25,7 +23,7 @@
public class DataExportService {

private final FileDefinitionEntityRepository fileDefinitionEntityRepository;
private final JobExecutionEntityRepository jobExecutionEntityRepository;
private final JobExecutionService jobExecutionService;
private final JobProfileEntityRepository jobProfileEntityRepository;
private final ExportIdEntityRepository exportIdEntityRepository;
private final InputFileProcessor inputFileProcessor;
Expand All @@ -41,33 +39,31 @@ public void postDataExport(ExportRequest exportRequest) {
getReferenceById(exportRequest.getFileDefinitionId());
var fileDefinition = fileDefinitionEntity.getFileDefinition();
var jobProfileEntity = jobProfileEntityRepository.getReferenceById(exportRequest.getJobProfileId());
var jobExecutionEntity = jobExecutionEntityRepository.getReferenceById(fileDefinition.getJobExecutionId());
jobExecutionEntity.getJobExecution().setJobProfileId(jobProfileEntity.getJobProfile().getId());
jobExecutionEntity.getJobExecution().setJobProfileName(jobProfileEntity.getJobProfile().getName());
jobExecutionEntity.setJobProfileId(jobProfileEntity.getId());
var jobExecution = jobExecutionService.getById(fileDefinition.getJobExecutionId());
jobExecution.setJobProfileId(jobProfileEntity.getJobProfile().getId());
jobExecution.setJobProfileName(jobProfileEntity.getJobProfile().getName());
try {
dataExportRequestValidator.validate(exportRequest, fileDefinition, jobProfileEntity.getJobProfile().getMappingProfileId().toString());
} catch (DataExportRequestValidationException e) {
updateJobExecutionForPostDataExport(jobExecutionEntity, JobExecution.StatusEnum.FAIL, commonExportFails);
updateJobExecutionForPostDataExport(jobExecution, JobExecution.StatusEnum.FAIL, commonExportFails);
log.error(e.getMessage());
return;
}
log.info("Post data export{} for file definition {} and job profile {} with job execution {}",
Boolean.TRUE.equals(exportRequest.getAll()) ? " all" : "", exportRequest.getFileDefinitionId(), exportRequest.getJobProfileId(), jobExecutionEntity.getId());
Boolean.TRUE.equals(exportRequest.getAll()) ? " all" : "", exportRequest.getFileDefinitionId(), exportRequest.getJobProfileId(), jobExecution.getId());

if (!Boolean.TRUE.equals(exportRequest.getAll())) {
if (Boolean.FALSE.equals(exportRequest.getAll()) && Boolean.FALSE.equals(exportRequest.getQuick())) {
inputFileProcessor.readFile(fileDefinition, commonExportFails);
log.info("File has been read successfully.");
}
slicerProcessor.sliceInstancesIds(fileDefinition, exportRequest);
log.info("Instance IDs have been sliced successfully.");

updateJobExecutionForPostDataExport(jobExecutionEntity, JobExecution.StatusEnum.IN_PROGRESS, commonExportFails);
singleFileProcessor.exportBySingleFile(jobExecutionEntity.getId(), exportRequest, commonExportFails);
updateJobExecutionForPostDataExport(jobExecution, JobExecution.StatusEnum.IN_PROGRESS, commonExportFails);
singleFileProcessor.exportBySingleFile(jobExecution.getId(), exportRequest, commonExportFails);
}

private void updateJobExecutionForPostDataExport(JobExecutionEntity jobExecutionEntity, JobExecution.StatusEnum jobExecutionStatus, CommonExportFails commonExportFails) {
var jobExecution = jobExecutionEntity.getJobExecution();
private void updateJobExecutionForPostDataExport(JobExecution jobExecution, JobExecution.StatusEnum jobExecutionStatus, CommonExportFails commonExportFails) {
jobExecution.setStatus(jobExecutionStatus);
var currentDate = new Date();
jobExecution.setStartedDate(currentDate);
Expand All @@ -89,10 +85,8 @@ private void updateJobExecutionForPostDataExport(JobExecutionEntity jobExecution
jobExecutionProgress.setTotal((int) totalExportsIds + commonExportFails.getDuplicatedUUIDAmount() + commonExportFails.getInvalidUUIDFormat().size());
jobExecution.setProgress(jobExecutionProgress);

int hrid = jobExecutionEntityRepository.getHrid();
jobExecution.setHrId(hrid);
jobExecution.setHrId(jobExecutionService.getNextHrid());

jobExecutionEntity.setStatus(jobExecution.getStatus());
jobExecutionEntityRepository.save(jobExecutionEntity);
jobExecutionService.save(jobExecution);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,8 @@
import org.folio.dataexp.domain.dto.JobExecution;
import org.folio.dataexp.domain.dto.Metadata;
import org.folio.dataexp.domain.entity.FileDefinitionEntity;
import org.folio.dataexp.domain.entity.JobExecutionEntity;
import org.folio.dataexp.exception.file.definition.UploadFileException;
import org.folio.dataexp.repository.FileDefinitionEntityRepository;
import org.folio.dataexp.repository.JobExecutionEntityRepository;
import org.folio.dataexp.service.file.upload.FilesUploadService;
import org.folio.dataexp.service.validators.FileDefinitionValidator;
import org.folio.spring.FolioExecutionContext;
Expand All @@ -25,19 +23,15 @@
@Log4j2
public class FileDefinitionsService {
private final FileDefinitionEntityRepository fileDefinitionEntityRepository;
private final JobExecutionEntityRepository jobExecutionEntityRepository;
private final JobExecutionService jobExecutionService;
private final FilesUploadService filesUploadService;
private final FileDefinitionValidator fileDefinitionValidator;
private final FolioExecutionContext folioExecutionContext;

public FileDefinition postFileDefinition(FileDefinition fileDefinition) {
log.info("Post file definition by id {}", fileDefinition.getId());
fileDefinitionValidator.validate(fileDefinition);
var jobExecution = new JobExecution();
jobExecution.setId(UUID.randomUUID());
jobExecution.setStatus(JobExecution.StatusEnum.NEW);
jobExecutionEntityRepository.save(JobExecutionEntity.builder()
.id(jobExecution.getId()).jobExecution(jobExecution).build());
var jobExecution = jobExecutionService.save(new JobExecution().status(JobExecution.StatusEnum.NEW));
fileDefinition.setJobExecutionId(jobExecution.getId());
fileDefinition.setStatus(FileDefinition.StatusEnum.NEW);
var now = new Date();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import org.folio.dataexp.domain.entity.JobExecutionEntity;
import org.folio.dataexp.repository.ErrorLogEntityCqlRepository;
import org.folio.dataexp.repository.JobExecutionEntityCqlRepository;
import org.folio.dataexp.repository.JobExecutionEntityRepository;
import org.springframework.stereotype.Service;

import java.util.Date;
Expand All @@ -22,6 +23,7 @@
@RequiredArgsConstructor
public class JobExecutionService {
private final JobExecutionEntityCqlRepository jobExecutionEntityCqlRepository;
private final JobExecutionEntityRepository jobExecutionEntityRepository;
private final ErrorLogEntityCqlRepository errorLogEntityCqlRepository;

public JobExecution getById(UUID id) {
Expand All @@ -32,7 +34,11 @@ public JobExecution save(JobExecution jobExecution) {
if (isNull(jobExecution.getId())) {
jobExecution.setId(UUID.randomUUID());
}
return jobExecutionEntityCqlRepository.save(JobExecutionEntity.from(jobExecution)).getJobExecution();
return jobExecutionEntityCqlRepository.save(JobExecutionEntity.fromJobExecution(jobExecution)).getJobExecution();
}

public int getNextHrid() {
return jobExecutionEntityRepository.getHrid();
}

public void expireJobExecutions() {
Expand Down
79 changes: 79 additions & 0 deletions src/main/java/org/folio/dataexp/service/QuickExportService.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
package org.folio.dataexp.service;

import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.folio.dataexp.domain.dto.ExportRequest;
import org.folio.dataexp.domain.dto.FileDefinition;
import org.folio.dataexp.domain.dto.QuickExportRequest;
import org.folio.dataexp.domain.dto.QuickExportResponse;
import org.folio.dataexp.domain.entity.ExportIdEntity;
import org.folio.dataexp.exception.export.DataExportRequestValidationException;
import org.folio.dataexp.repository.ExportIdEntityRepository;
import org.folio.dataexp.repository.JobProfileEntityRepository;
import org.springframework.stereotype.Service;

import java.util.ArrayList;
import java.util.UUID;

import static java.util.Objects.nonNull;

@Service
@RequiredArgsConstructor
@Log4j2
public class QuickExportService {

private final FileDefinitionsService fileDefinitionsService;
private final DataExportService dataExportService;
private final JobProfileEntityRepository jobProfileEntityRepository;
private final ExportIdEntityRepository exportIdEntityRepository;
private final JobExecutionService jobExecutionService;

public QuickExportResponse postQuickExport(QuickExportRequest quickExportRequest) {
var fileDefinition = new FileDefinition().id(UUID.randomUUID()).size(0).fileName("quick-export.csv");
fileDefinitionsService.postFileDefinition(fileDefinition);
log.info("Post quick export for job profile {}", quickExportRequest.getJobProfileId());
dataExportService.postDataExport(getExportRequestFromQuickExportRequest(quickExportRequest, fileDefinition));
var jobExecution = jobExecutionService.getById(fileDefinition.getJobExecutionId());
return QuickExportResponse.builder().jobExecutionId(fileDefinition.getJobExecutionId())
.jobExecutionHrId(jobExecution.getHrId()).build();
}

private ExportRequest getExportRequestFromQuickExportRequest(QuickExportRequest quickExportRequest, FileDefinition fileDefinition) {
var exportRequest = new ExportRequest();
saveBatch(quickExportRequest, fileDefinition);
exportRequest.setJobProfileId(getDefaultJobProfileId(quickExportRequest));
exportRequest.setRecordType(ExportRequest.RecordTypeEnum.fromValue(quickExportRequest.getRecordType().getValue()));
exportRequest.setFileDefinitionId(fileDefinition.getId());
exportRequest.setQuick(true);
exportRequest.setIdType(ExportRequest.IdTypeEnum.fromValue(quickExportRequest.getRecordType().getValue().toLowerCase()));
return exportRequest;
}

private void saveBatch(QuickExportRequest quickExportRequest, FileDefinition fileDefinition) {
var uuids = quickExportRequest.getUuids();
if (nonNull(uuids)) {
var batch = new ArrayList<ExportIdEntity>();
uuids.forEach(instanceId -> {
var entity = ExportIdEntity.builder().jobExecutionId(fileDefinition
.getJobExecutionId()).instanceId(instanceId).build();
batch.add(entity);
});
exportIdEntityRepository.saveAll(batch);
} else {
log.error("Nothing to export for fileDefinitionId {}: no uuids provided.", fileDefinition.getId());
}
}

private UUID getDefaultJobProfileId(QuickExportRequest quickExportRequest) {
var ids = jobProfileEntityRepository.findIdOfDefaultJobProfileByName(quickExportRequest.getRecordType().getValue().toLowerCase());
if (ids.isEmpty()) {
log.error("No default job profile found by the following recordType: {}", quickExportRequest.getRecordType());
throw new DataExportRequestValidationException("No default job profile found by the following recordType: " + quickExportRequest.getRecordType());
}
if (ids.size() > 1) {
log.warn("More than 1 job profile found by the following recordType: {}, only first one will be used: {}",
quickExportRequest.getRecordType(), ids.get(0));
}
return ids.get(0);
}
}
15 changes: 5 additions & 10 deletions src/main/java/org/folio/dataexp/service/SingleFileProcessor.java
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
import org.folio.dataexp.domain.dto.JobExecution;
import org.folio.dataexp.domain.entity.JobExecutionExportFilesEntity;
import org.folio.dataexp.exception.export.DataExportException;
import org.folio.dataexp.repository.JobExecutionEntityRepository;
import org.folio.dataexp.repository.JobExecutionExportFilesEntityRepository;
import org.folio.dataexp.service.export.ExportExecutor;
import org.folio.dataexp.service.logs.ErrorLogService;
Expand All @@ -27,27 +26,24 @@ public class SingleFileProcessor {

protected final ExportExecutor exportExecutor;
private final JobExecutionExportFilesEntityRepository jobExecutionExportFilesEntityRepository;
private final JobExecutionEntityRepository jobExecutionEntityRepository;
private final JobExecutionService jobExecutionService;
private final ErrorLogService errorLogService;

public void exportBySingleFile(UUID jobExecutionId, ExportRequest exportRequest, CommonExportFails commonExportFails) {
var exports = jobExecutionExportFilesEntityRepository.findByJobExecutionId(jobExecutionId);
if (exports.isEmpty()) {
log.error("Nothing to export for job execution {}", jobExecutionId);
var jobExecutionEntity = jobExecutionEntityRepository.getReferenceById(jobExecutionId);
var jobExecution = jobExecutionEntity.getJobExecution();
var jobExecution = jobExecutionService.getById(jobExecutionId);
var currentDate = new Date();
jobExecution.setLastUpdatedDate(currentDate);
jobExecution.setStatus(JobExecution.StatusEnum.FAIL);
jobExecution.setCompletedDate(currentDate);

jobExecutionEntity.setStatus(jobExecution.getStatus());
jobExecutionEntity.setCompletedDate(jobExecution.getCompletedDate());
var totalFailed = commonExportFails.getInvalidUUIDFormat().size();
var progress = jobExecution.getProgress();
progress.setFailed(totalFailed);
progress.setExported(0);
jobExecutionEntityRepository.save(jobExecutionEntity);
jobExecutionService.save(jobExecution);
if (commonExportFails.isFailedToReadInputFile()) {
errorLogService.saveFailedToReadInputFileError(jobExecutionId);
} else {
Expand Down Expand Up @@ -77,10 +73,9 @@ public void executeExport(JobExecutionExportFilesEntity export, ExportRequest ex
}

private void updateStatisticsForExportAll(UUID jobExecutionId) {
var jobExecutionEntity = jobExecutionEntityRepository.getReferenceById(jobExecutionId);
var jobExecution = jobExecutionEntity.getJobExecution();
var jobExecution = jobExecutionService.getById(jobExecutionId);
var progress = jobExecution.getProgress();
progress.setTotal(progress.getExported() - progress.getDuplicatedSrs() + progress.getFailed());
jobExecutionEntityRepository.save(jobExecutionEntity);
jobExecutionService.save(jobExecution);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

import org.folio.dataexp.domain.dto.ExportRequest;
import org.folio.dataexp.domain.entity.JobExecutionExportFilesEntity;
import org.folio.dataexp.repository.JobExecutionEntityRepository;
import org.folio.dataexp.repository.JobExecutionExportFilesEntityRepository;
import org.folio.dataexp.service.export.ExportExecutor;
import org.folio.dataexp.service.logs.ErrorLogService;
Expand All @@ -15,8 +14,8 @@ public class SingleFileProcessorAsync extends SingleFileProcessor {

@Autowired
public SingleFileProcessorAsync(ExportExecutor exportExecutor, JobExecutionExportFilesEntityRepository jobExecutionExportFilesEntityRepository,
JobExecutionEntityRepository jobExecutionEntityRepository, ErrorLogService errorLogService) {
super(exportExecutor, jobExecutionExportFilesEntityRepository, jobExecutionEntityRepository, errorLogService);
JobExecutionService jobExecutionService, ErrorLogService errorLogService) {
super(exportExecutor, jobExecutionExportFilesEntityRepository, jobExecutionService, errorLogService);
}

@Override
Expand Down
Loading

0 comments on commit 7261438

Please sign in to comment.