From 29387d344b2196a6fdebbe1a5efa3653b895d73d Mon Sep 17 00:00:00 2001 From: Aliaksei_Harbuz Date: Fri, 1 Sep 2023 14:26:53 +0300 Subject: [PATCH] MDEXP-630 - single file processor --- .gitignore | 1 + .../controllers/DataExportController.java | 17 +++++- .../domain/entity/JobExecutionEntity.java | 2 + .../entity/JobExecutionExportFilesEntity.java | 5 +- .../entity/JobExecutionExportFilesStatus.java | 5 ++ .../exception/DataExportExceptionHandler.java | 6 ++ .../exception/export/DataExportException.java | 7 +++ ...bExecutionExportFilesEntityRepository.java | 12 ++++ .../JobExecutionExportFilesRepository.java | 9 --- .../dataexp/service/DataExportService.java | 53 ++++++++-------- .../service/FileDefinitionValidator.java | 40 ++++++++++++ .../dataexp/service/InputFileProcessor.java | 12 +++- .../dataexp/service/SingleFileProcessor.java | 61 +++++++++++++++++++ .../dataexp/service/SlicerProcessor.java | 1 + .../export/strategies/ExportStrategy.java | 11 ++++ .../strategies/InstancesExportStrategy.java | 23 +++++++ .../resources/swagger.api/data-export.yaml | 2 +- src/test/java/org/folio/dataexp/BaseTest.java | 11 +++- .../service/DataExportServiceTest.java | 22 +------ .../service/FileDefinitionValidatorTest.java | 34 +++++++++++ .../service/InputFileProcessorTest.java | 6 +- .../service/SingleFileProcessorTest.java | 58 ++++++++++++++++++ .../dataexp/service/SlicerProcessorTest.java | 22 ++++--- 23 files changed, 347 insertions(+), 73 deletions(-) create mode 100644 src/main/java/org/folio/dataexp/domain/entity/JobExecutionExportFilesStatus.java create mode 100644 src/main/java/org/folio/dataexp/exception/export/DataExportException.java create mode 100644 src/main/java/org/folio/dataexp/repository/JobExecutionExportFilesEntityRepository.java delete mode 100644 src/main/java/org/folio/dataexp/repository/JobExecutionExportFilesRepository.java create mode 100644 src/main/java/org/folio/dataexp/service/FileDefinitionValidator.java create mode 100644 src/main/java/org/folio/dataexp/service/SingleFileProcessor.java create mode 100644 src/main/java/org/folio/dataexp/service/export/strategies/ExportStrategy.java create mode 100644 src/main/java/org/folio/dataexp/service/export/strategies/InstancesExportStrategy.java create mode 100644 src/test/java/org/folio/dataexp/service/FileDefinitionValidatorTest.java create mode 100644 src/test/java/org/folio/dataexp/service/SingleFileProcessorTest.java diff --git a/.gitignore b/.gitignore index 549e00a2a..a5a07d84b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ HELP.md target/ +mod-data-export/ !.mvn/wrapper/maven-wrapper.jar !**/src/main/**/target/ !**/src/test/**/target/ diff --git a/src/main/java/org/folio/dataexp/controllers/DataExportController.java b/src/main/java/org/folio/dataexp/controllers/DataExportController.java index af665873d..92a0a51ef 100644 --- a/src/main/java/org/folio/dataexp/controllers/DataExportController.java +++ b/src/main/java/org/folio/dataexp/controllers/DataExportController.java @@ -2,7 +2,9 @@ import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; +import org.folio.dataexp.domain.dto.ExportRequest; import org.folio.dataexp.domain.dto.FileDefinition; +import org.folio.dataexp.rest.resource.ExportApi; import org.folio.dataexp.rest.resource.FileDefinitionsApi; import org.folio.dataexp.service.DataExportService; import org.springframework.core.io.Resource; @@ -10,7 +12,9 @@ import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.context.request.NativeWebRequest; +import java.util.Optional; import java.util.UUID; @@ -18,7 +22,7 @@ @RequiredArgsConstructor @Log4j2 @RequestMapping("/data-export") -public class DataExportController implements FileDefinitionsApi { +public class DataExportController implements FileDefinitionsApi, ExportApi { private final DataExportService dataExportService; @@ -38,4 +42,15 @@ public ResponseEntity uploadFile(UUID fileDefinitionId, Resource var fileDefinition = dataExportService.uploadFile(fileDefinitionId, resource); return new ResponseEntity<>(fileDefinition, HttpStatus.OK); } + + @Override + public ResponseEntity postDataExport(ExportRequest exportRequest) { + dataExportService.postDataExport(exportRequest); + return new ResponseEntity<>(HttpStatus.OK); + } + + @Override + public Optional getRequest() { + return Optional.empty(); + } } diff --git a/src/main/java/org/folio/dataexp/domain/entity/JobExecutionEntity.java b/src/main/java/org/folio/dataexp/domain/entity/JobExecutionEntity.java index d3837f017..cf2a68ccd 100644 --- a/src/main/java/org/folio/dataexp/domain/entity/JobExecutionEntity.java +++ b/src/main/java/org/folio/dataexp/domain/entity/JobExecutionEntity.java @@ -7,6 +7,7 @@ import jakarta.persistence.Table; import lombok.AllArgsConstructor; import lombok.Builder; +import lombok.Data; import lombok.NoArgsConstructor; import lombok.With; import org.folio.dataexp.domain.dto.JobExecution; @@ -14,6 +15,7 @@ import java.util.UUID; +@Data @Builder @With @AllArgsConstructor diff --git a/src/main/java/org/folio/dataexp/domain/entity/JobExecutionExportFilesEntity.java b/src/main/java/org/folio/dataexp/domain/entity/JobExecutionExportFilesEntity.java index fff9e3254..0fc597f48 100644 --- a/src/main/java/org/folio/dataexp/domain/entity/JobExecutionExportFilesEntity.java +++ b/src/main/java/org/folio/dataexp/domain/entity/JobExecutionExportFilesEntity.java @@ -1,6 +1,8 @@ package org.folio.dataexp.domain.entity; import jakarta.persistence.Entity; +import jakarta.persistence.EnumType; +import jakarta.persistence.Enumerated; import jakarta.persistence.Id; import jakarta.persistence.Table; import lombok.AllArgsConstructor; @@ -31,5 +33,6 @@ public class JobExecutionExportFilesEntity { private UUID toId; - private String status; + @Enumerated(EnumType.STRING) + private JobExecutionExportFilesStatus status; } diff --git a/src/main/java/org/folio/dataexp/domain/entity/JobExecutionExportFilesStatus.java b/src/main/java/org/folio/dataexp/domain/entity/JobExecutionExportFilesStatus.java new file mode 100644 index 000000000..fa7701ec0 --- /dev/null +++ b/src/main/java/org/folio/dataexp/domain/entity/JobExecutionExportFilesStatus.java @@ -0,0 +1,5 @@ +package org.folio.dataexp.domain.entity; + +public enum JobExecutionExportFilesStatus { + SCHEDULED, ACTIVE, COMPLETED, FAILED; +} diff --git a/src/main/java/org/folio/dataexp/exception/DataExportExceptionHandler.java b/src/main/java/org/folio/dataexp/exception/DataExportExceptionHandler.java index c9cfe4531..9d634a7c6 100644 --- a/src/main/java/org/folio/dataexp/exception/DataExportExceptionHandler.java +++ b/src/main/java/org/folio/dataexp/exception/DataExportExceptionHandler.java @@ -1,6 +1,7 @@ package org.folio.dataexp.exception; import jakarta.persistence.EntityNotFoundException; +import org.folio.dataexp.exception.export.DataExportException; import org.folio.dataexp.exception.export.FileExtensionException; import org.folio.dataexp.exception.export.FileSizeException; import org.folio.dataexp.exception.export.UploadFileException; @@ -31,4 +32,9 @@ public ResponseEntity handleUploadFileException(final UploadFileExceptio public ResponseEntity handleEntityNotFoundException(final EntityNotFoundException e) { return new ResponseEntity<>(e.getMessage(), HttpStatus.NOT_FOUND); } + + @ExceptionHandler(DataExportException.class) + public ResponseEntity handleDataExportException(final DataExportException e) { + return new ResponseEntity<>(e.getMessage(), HttpStatus.INTERNAL_SERVER_ERROR); + } } diff --git a/src/main/java/org/folio/dataexp/exception/export/DataExportException.java b/src/main/java/org/folio/dataexp/exception/export/DataExportException.java new file mode 100644 index 000000000..7b8ad9878 --- /dev/null +++ b/src/main/java/org/folio/dataexp/exception/export/DataExportException.java @@ -0,0 +1,7 @@ +package org.folio.dataexp.exception.export; + +public class DataExportException extends RuntimeException{ + public DataExportException(String message) { + super(message); + } +} diff --git a/src/main/java/org/folio/dataexp/repository/JobExecutionExportFilesEntityRepository.java b/src/main/java/org/folio/dataexp/repository/JobExecutionExportFilesEntityRepository.java new file mode 100644 index 000000000..de262148c --- /dev/null +++ b/src/main/java/org/folio/dataexp/repository/JobExecutionExportFilesEntityRepository.java @@ -0,0 +1,12 @@ +package org.folio.dataexp.repository; + +import org.folio.dataexp.domain.entity.JobExecutionExportFilesEntity; +import org.springframework.data.jpa.repository.JpaRepository; + +import java.util.List; +import java.util.UUID; + +public interface JobExecutionExportFilesEntityRepository extends JpaRepository { + + List findByJobExecutionId(UUID jobExecutionId); +} diff --git a/src/main/java/org/folio/dataexp/repository/JobExecutionExportFilesRepository.java b/src/main/java/org/folio/dataexp/repository/JobExecutionExportFilesRepository.java deleted file mode 100644 index 0b184b875..000000000 --- a/src/main/java/org/folio/dataexp/repository/JobExecutionExportFilesRepository.java +++ /dev/null @@ -1,9 +0,0 @@ -package org.folio.dataexp.repository; - -import org.folio.dataexp.domain.entity.JobExecutionExportFilesEntity; -import org.springframework.data.jpa.repository.JpaRepository; - -import java.util.UUID; - -public interface JobExecutionExportFilesRepository extends JpaRepository { -} diff --git a/src/main/java/org/folio/dataexp/service/DataExportService.java b/src/main/java/org/folio/dataexp/service/DataExportService.java index 92c3b734e..505f68b66 100644 --- a/src/main/java/org/folio/dataexp/service/DataExportService.java +++ b/src/main/java/org/folio/dataexp/service/DataExportService.java @@ -2,23 +2,23 @@ import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; -import org.apache.commons.io.FilenameUtils; +import org.folio.dataexp.domain.dto.ExportRequest; import org.folio.dataexp.domain.dto.FileDefinition; import org.folio.dataexp.domain.dto.JobExecution; import org.folio.dataexp.domain.entity.FileDefinitionEntity; import org.folio.dataexp.domain.entity.JobExecutionEntity; -import org.folio.dataexp.exception.export.FileExtensionException; -import org.folio.dataexp.exception.export.FileSizeException; +import org.folio.dataexp.exception.export.DataExportException; import org.folio.dataexp.exception.export.UploadFileException; import org.folio.dataexp.repository.FileDefinitionEntityRepository; import org.folio.dataexp.repository.JobExecutionEntityRepository; +import org.folio.dataexp.repository.JobExecutionExportFilesEntityRepository; +import org.folio.dataexp.repository.JobProfileEntityRepository; import org.folio.dataexp.service.file.upload.FileUploadService; import org.folio.spring.FolioExecutionContext; import org.springframework.core.io.Resource; import org.springframework.stereotype.Service; import java.time.LocalDateTime; -import java.util.Objects; import java.util.UUID; @Service @@ -26,31 +26,20 @@ @Log4j2 public class DataExportService { - private static final String CSV_FORMAT_EXTENSION = "csv"; - private static final String CQL_FORMAT_EXTENSION = "cql"; - - private static final int MAX_FILE_SIZE = 500_000; - private final FileDefinitionEntityRepository fileDefinitionEntityRepository; private final JobExecutionEntityRepository jobExecutionEntityRepository; + private final JobProfileEntityRepository jobProfileEntityRepository; + private final JobExecutionExportFilesEntityRepository jobExecutionExportFilesEntityRepository; private final FileUploadService fileUploadService; + private final InputFileProcessor inputFileProcessor; + private final SlicerProcessor slicerProcessor; + private final SingleFileProcessor singleFileProcessor; + private final FileDefinitionValidator fileDefinitionValidator; private final FolioExecutionContext folioExecutionContext; public FileDefinition postFileDefinition(FileDefinition fileDefinition) { log.info("Post file definition by id {}", fileDefinition.getId()); - if (Objects.nonNull(fileDefinition.getSize()) && fileDefinition.getSize() > MAX_FILE_SIZE) { - var errorMessage = String.format("File size is too large: '%d'. Please use file with size less than %d.", fileDefinition.getSize(), MAX_FILE_SIZE); - log.error(errorMessage); - throw new FileSizeException(errorMessage); - } - if (Objects.isNull(fileDefinition.getSize())) { - log.error("Size of uploading file is null."); - } - if (isNotValidFileNameExtension(fileDefinition.getFileName())) { - var errorMessage = String.format("Incorrect file extension of %s", fileDefinition.getFileName()); - log.error(errorMessage); - throw new FileExtensionException(errorMessage); - } + fileDefinitionValidator.validate(fileDefinition); var jobExecution = new JobExecution(); jobExecution.setId(UUID.randomUUID()); jobExecutionEntityRepository.save(JobExecutionEntity.builder() @@ -79,7 +68,23 @@ public FileDefinition uploadFile(UUID fileDefinitionId, Resource resource) { } } - private boolean isNotValidFileNameExtension(String fileName) { - return !FilenameUtils.isExtension(fileName.toLowerCase(), CSV_FORMAT_EXTENSION) && !FilenameUtils.isExtension(fileName.toLowerCase(), CQL_FORMAT_EXTENSION); + public void postDataExport(ExportRequest exportRequest) { + log.info("Post data export for file definition {} and job profile {}", exportRequest.getFileDefinitionId(), exportRequest.getJobProfileId()); + var fileDefinition = fileDefinitionEntityRepository. + getReferenceById(exportRequest.getFileDefinitionId()).getFileDefinition(); + var jobProfileEntity = jobProfileEntityRepository.getReferenceById(exportRequest.getJobProfileId()); + var jobExecutionEntity = jobExecutionEntityRepository.getReferenceById(fileDefinition.getJobExecutionId()); + var jobExecution = jobExecutionEntity.getJobExecution(); + jobExecution.setJobProfileId(jobProfileEntity.getJobProfile().getId()); + jobExecution.setJobProfileName(jobProfileEntity.getJobProfile().getName()); + jobExecutionEntityRepository.save(jobExecutionEntity); + try { + inputFileProcessor.readFile(fileDefinition); + slicerProcessor.sliceInstancesIds(fileDefinition); + } catch (Exception e) { + throw new DataExportException(e.getMessage()); + } + var jobExecutionExportFilesEntities = jobExecutionExportFilesEntityRepository.findByJobExecutionId(jobExecution.getId()); + singleFileProcessor.exportBySingleFile(jobExecutionExportFilesEntities, exportRequest.getRecordType()); } } diff --git a/src/main/java/org/folio/dataexp/service/FileDefinitionValidator.java b/src/main/java/org/folio/dataexp/service/FileDefinitionValidator.java new file mode 100644 index 000000000..c601d84f5 --- /dev/null +++ b/src/main/java/org/folio/dataexp/service/FileDefinitionValidator.java @@ -0,0 +1,40 @@ +package org.folio.dataexp.service; + +import lombok.extern.log4j.Log4j2; +import org.apache.commons.io.FilenameUtils; +import org.folio.dataexp.domain.dto.FileDefinition; +import org.folio.dataexp.exception.export.FileExtensionException; +import org.folio.dataexp.exception.export.FileSizeException; +import org.springframework.stereotype.Component; + +import java.util.Objects; + +@Component +@Log4j2 +public class FileDefinitionValidator { + + private static final String CSV_FORMAT_EXTENSION = "csv"; + private static final String CQL_FORMAT_EXTENSION = "cql"; + + private static final int MAX_FILE_SIZE = 500_000; + + public void validate(FileDefinition fileDefinition) { + if (Objects.nonNull(fileDefinition.getSize()) && fileDefinition.getSize() > MAX_FILE_SIZE) { + var errorMessage = String.format("File size is too large: '%d'. Please use file with size less than %d.", fileDefinition.getSize(), MAX_FILE_SIZE); + log.error(errorMessage); + throw new FileSizeException(errorMessage); + } + if (Objects.isNull(fileDefinition.getSize())) { + log.error("Size of uploading file is null."); + } + if (isNotValidFileNameExtension(fileDefinition.getFileName())) { + var errorMessage = String.format("Incorrect file extension of %s", fileDefinition.getFileName()); + log.error(errorMessage); + throw new FileExtensionException(errorMessage); + } + } + + private boolean isNotValidFileNameExtension(String fileName) { + return !FilenameUtils.isExtension(fileName.toLowerCase(), CSV_FORMAT_EXTENSION) && !FilenameUtils.isExtension(fileName.toLowerCase(), CQL_FORMAT_EXTENSION); + } +} diff --git a/src/main/java/org/folio/dataexp/service/InputFileProcessor.java b/src/main/java/org/folio/dataexp/service/InputFileProcessor.java index f981f3cba..281573f2a 100644 --- a/src/main/java/org/folio/dataexp/service/InputFileProcessor.java +++ b/src/main/java/org/folio/dataexp/service/InputFileProcessor.java @@ -40,7 +40,15 @@ public class InputFileProcessor { private final FolioS3ClientFactory folioS3ClientFactory; private final JdbcTemplate jdbcTemplate; - public void readCsvFile(FileDefinition fileDefinition) throws IOException { + public void readFile(FileDefinition fileDefinition) throws Exception { + if (fileDefinition.getUploadFormat() == FileDefinition.UploadFormatEnum.CQL) { + readCqlFile(fileDefinition); + return; + } + readCsvFile(fileDefinition); + } + + private void readCsvFile(FileDefinition fileDefinition) throws IOException { var pathToRead = getPathToRead(fileDefinition); var s3Client = folioS3ClientFactory.getFolioS3Client(); try (InputStream is = s3Client.read(pathToRead); BufferedReader reader = new BufferedReader(new InputStreamReader(is))) { @@ -53,7 +61,7 @@ public void readCsvFile(FileDefinition fileDefinition) throws IOException { } } - public void readCqlFile(FileDefinition fileDefinition) throws IOException, ServerChoiceIndexesException, FieldException, QueryValidationException, SQLException { + private void readCqlFile(FileDefinition fileDefinition) throws IOException, ServerChoiceIndexesException, FieldException, QueryValidationException, SQLException { var pathToRead = getPathToRead(fileDefinition); var s3Client = folioS3ClientFactory.getFolioS3Client(); String cql; diff --git a/src/main/java/org/folio/dataexp/service/SingleFileProcessor.java b/src/main/java/org/folio/dataexp/service/SingleFileProcessor.java new file mode 100644 index 000000000..03ed13d62 --- /dev/null +++ b/src/main/java/org/folio/dataexp/service/SingleFileProcessor.java @@ -0,0 +1,61 @@ +package org.folio.dataexp.service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.log4j.Log4j2; +import org.folio.dataexp.domain.dto.ExportRequest; +import org.folio.dataexp.domain.entity.JobExecutionExportFilesEntity; +import org.folio.dataexp.domain.entity.JobExecutionExportFilesStatus; +import org.folio.dataexp.repository.JobExecutionExportFilesEntityRepository; +import org.folio.dataexp.service.export.storage.FolioS3ClientFactory; +import org.folio.dataexp.service.export.strategies.ExportStrategy; +import org.springframework.stereotype.Component; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; + +@Component +@RequiredArgsConstructor +@Log4j2 +public class SingleFileProcessor { + + private final ExportStrategy instanceExportStrategy; + private final FolioS3ClientFactory folioS3ClientFactory; + private final JobExecutionExportFilesEntityRepository jobExecutionExportFilesEntityRepository; + + public void exportBySingleFile(List exports, ExportRequest.RecordTypeEnum recordType) { + exports.forEach(export -> { + try { + createExportFile(export, recordType); + } catch (IOException e) { + export.setStatus(JobExecutionExportFilesStatus.FAILED); + jobExecutionExportFilesEntityRepository.save(export); + log.error("Error exporting {} for jobExecution {}", export.getFileLocation(), export.getJobExecutionId()); + } + }); + } + + private void createExportFile(JobExecutionExportFilesEntity exportFilesEntity, ExportRequest.RecordTypeEnum recordType) throws IOException { + exportFilesEntity.setStatus(JobExecutionExportFilesStatus.ACTIVE); + jobExecutionExportFilesEntityRepository.save(exportFilesEntity); + var file = new File(exportFilesEntity.getFileLocation()); + var parent = file.getParentFile(); + if (!parent.exists()) { + boolean isCreated = parent.mkdirs(); + if (isCreated) log.info("Create local directories to store {} ", exportFilesEntity.getFileLocation()); + } + boolean isCreated = file.createNewFile(); + if (isCreated) log.info("Create {} locally", exportFilesEntity.getFileLocation()); + instanceExportStrategy.saveMarc(exportFilesEntity, file); + var s3Client = folioS3ClientFactory.getFolioS3Client(); + try (InputStream is = new BufferedInputStream(new FileInputStream(file))) { + s3Client.write(exportFilesEntity.getFileLocation(), is); + log.info("Create {} at remote storage", exportFilesEntity.getFileLocation()); + } + exportFilesEntity.setStatus(JobExecutionExportFilesStatus.COMPLETED); + jobExecutionExportFilesEntityRepository.save(exportFilesEntity); + } +} diff --git a/src/main/java/org/folio/dataexp/service/SlicerProcessor.java b/src/main/java/org/folio/dataexp/service/SlicerProcessor.java index 65236278b..262693b8a 100644 --- a/src/main/java/org/folio/dataexp/service/SlicerProcessor.java +++ b/src/main/java/org/folio/dataexp/service/SlicerProcessor.java @@ -15,6 +15,7 @@ @RequiredArgsConstructor @Log4j2 public class SlicerProcessor { + private static final String CALL_SLICE_INSTANCES_IDS_PROCEDURE = "call slice_instances_ids(?, ?, ?)"; private static final String SLICED_FILE_LOCATION_PATH = "mod-data-export/download/%s/"; private static final String FROM_TO_UUID_PART = "_%s_%s"; diff --git a/src/main/java/org/folio/dataexp/service/export/strategies/ExportStrategy.java b/src/main/java/org/folio/dataexp/service/export/strategies/ExportStrategy.java new file mode 100644 index 000000000..a0e44a541 --- /dev/null +++ b/src/main/java/org/folio/dataexp/service/export/strategies/ExportStrategy.java @@ -0,0 +1,11 @@ +package org.folio.dataexp.service.export.strategies; + +import org.folio.dataexp.domain.entity.JobExecutionExportFilesEntity; + +import java.io.File; + +public interface ExportStrategy { + + void saveMarc(JobExecutionExportFilesEntity exportFilesEntity, File file); + +} diff --git a/src/main/java/org/folio/dataexp/service/export/strategies/InstancesExportStrategy.java b/src/main/java/org/folio/dataexp/service/export/strategies/InstancesExportStrategy.java new file mode 100644 index 000000000..7a0e40177 --- /dev/null +++ b/src/main/java/org/folio/dataexp/service/export/strategies/InstancesExportStrategy.java @@ -0,0 +1,23 @@ +package org.folio.dataexp.service.export.strategies; + +import org.apache.commons.io.FileUtils; +import org.folio.dataexp.domain.entity.JobExecutionExportFilesEntity; +import org.springframework.stereotype.Component; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; + +@Component +public class InstancesExportStrategy implements ExportStrategy { + + @Override + public void saveMarc(JobExecutionExportFilesEntity exportFilesEntity, File file) { + var marc = "marc"; + try { + FileUtils.writeStringToFile(file, marc, Charset.defaultCharset()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/src/main/resources/swagger.api/data-export.yaml b/src/main/resources/swagger.api/data-export.yaml index 6b41fcc69..e04896dac 100644 --- a/src/main/resources/swagger.api/data-export.yaml +++ b/src/main/resources/swagger.api/data-export.yaml @@ -9,7 +9,7 @@ paths: /export: post: description: Starts the export process - operationId: postDataExportExport + operationId: postDataExport requestBody: required: true content: diff --git a/src/test/java/org/folio/dataexp/BaseTest.java b/src/test/java/org/folio/dataexp/BaseTest.java index 852394940..9980072a8 100644 --- a/src/test/java/org/folio/dataexp/BaseTest.java +++ b/src/test/java/org/folio/dataexp/BaseTest.java @@ -10,7 +10,8 @@ import lombok.extern.log4j.Log4j2; import org.folio.dataexp.repository.ExportIdEntityRepository; import org.folio.dataexp.repository.JobExecutionEntityRepository; -import org.folio.dataexp.repository.JobExecutionExportFilesRepository; +import org.folio.dataexp.repository.JobExecutionExportFilesEntityRepository; +import org.folio.dataexp.service.export.storage.FolioS3ClientFactory; import org.folio.spring.DefaultFolioExecutionContext; import org.folio.spring.FolioExecutionContext; import org.folio.spring.FolioModuleMetadata; @@ -132,7 +133,9 @@ private static void runSqlScript(String path, JdbcTemplate jdbcTemplate) throws @Autowired private ExportIdEntityRepository exportIdEntityRepository; @Autowired - private JobExecutionExportFilesRepository jobExecutionExportFilesRepository; + private JobExecutionExportFilesEntityRepository jobExecutionExportFilesEntityRepository; + @Autowired + private FolioS3ClientFactory folioS3ClientFactory; public final Map okapiHeaders = new HashMap<>(); @@ -173,13 +176,15 @@ void setUp() { .collect(Collectors.toMap(Map.Entry::getKey, e -> (Collection) List.of(String.valueOf(e.getValue())))); folioExecutionContext = new DefaultFolioExecutionContext(folioModuleMetadata, localHeaders); + var s3Client = folioS3ClientFactory.getFolioS3Client(); + s3Client.createBucketIfNotExists(); } @AfterEach void eachTearDown() { try (var context = new FolioExecutionContextSetter(folioExecutionContext)) { exportIdEntityRepository.deleteAll(); - jobExecutionExportFilesRepository.deleteAll(); + jobExecutionExportFilesEntityRepository.deleteAll(); jobExecutionEntityRepository.deleteAll(); } } diff --git a/src/test/java/org/folio/dataexp/service/DataExportServiceTest.java b/src/test/java/org/folio/dataexp/service/DataExportServiceTest.java index 1088e2d9a..8fb03c013 100644 --- a/src/test/java/org/folio/dataexp/service/DataExportServiceTest.java +++ b/src/test/java/org/folio/dataexp/service/DataExportServiceTest.java @@ -32,6 +32,8 @@ public class DataExportServiceTest { private JobExecutionEntityRepository jobExecutionEntityRepository; @Mock private FolioExecutionContext folioExecutionContext; + @Mock + private FileDefinitionValidator fileDefinitionValidator; @InjectMocks private DataExportService dataExportService; @@ -53,24 +55,4 @@ void postFileDefinitionTest() { verify(fileDefinitionEntityRepository).save(isA(FileDefinitionEntity.class)); verify(jobExecutionEntityRepository).save(isA(JobExecutionEntity.class)); } - - @Test - void postFileDefinitionWithFileSizeExceptionTest() { - var fileDefinition = new FileDefinition(); - fileDefinition.setId(UUID.randomUUID()); - fileDefinition.fileName("upload.csv"); - fileDefinition.setSize(500_001); - - assertThrows(FileSizeException.class, () -> dataExportService.postFileDefinition(fileDefinition)); - } - - @Test - void postFileDefinitionWithFileExtensionExceptionTest() { - var fileDefinition = new FileDefinition(); - fileDefinition.setId(UUID.randomUUID()); - fileDefinition.fileName("upload.txt"); - - assertThrows(FileExtensionException.class, () -> dataExportService.postFileDefinition(fileDefinition)); - } - } diff --git a/src/test/java/org/folio/dataexp/service/FileDefinitionValidatorTest.java b/src/test/java/org/folio/dataexp/service/FileDefinitionValidatorTest.java new file mode 100644 index 000000000..c7bbb63f9 --- /dev/null +++ b/src/test/java/org/folio/dataexp/service/FileDefinitionValidatorTest.java @@ -0,0 +1,34 @@ +package org.folio.dataexp.service; + +import org.folio.dataexp.domain.dto.FileDefinition; +import org.folio.dataexp.exception.export.FileExtensionException; +import org.folio.dataexp.exception.export.FileSizeException; +import org.junit.jupiter.api.Test; + +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class FileDefinitionValidatorTest { + + @Test + void validateFileSizeTest() { + var validator = new FileDefinitionValidator(); + var fileDefinition = new FileDefinition(); + fileDefinition.setId(UUID.randomUUID()); + fileDefinition.fileName("upload.csv"); + fileDefinition.setSize(500_001); + + assertThrows(FileSizeException.class, () -> validator.validate(fileDefinition)); + } + + @Test + void validateFileExtensionTest() { + var validator = new FileDefinitionValidator(); + var fileDefinition = new FileDefinition(); + fileDefinition.setId(UUID.randomUUID()); + fileDefinition.fileName("upload.txt"); + + assertThrows(FileExtensionException.class, () -> validator.validate(fileDefinition)); + } +} diff --git a/src/test/java/org/folio/dataexp/service/InputFileProcessorTest.java b/src/test/java/org/folio/dataexp/service/InputFileProcessorTest.java index 32a0f21ef..9559aa023 100644 --- a/src/test/java/org/folio/dataexp/service/InputFileProcessorTest.java +++ b/src/test/java/org/folio/dataexp/service/InputFileProcessorTest.java @@ -37,6 +37,7 @@ void readCsvFileTest() { var fileDefinition = new FileDefinition(); fileDefinition.setId(UUID.randomUUID()); fileDefinition.fileName("upload.csv"); + fileDefinition.setUploadFormat(FileDefinition.UploadFormatEnum.CSV); fileDefinition.setJobExecutionId(UUID.randomUUID()); var s3Client = folioS3ClientFactory.getFolioS3Client(); @@ -49,7 +50,7 @@ void readCsvFileTest() { var jobExecutionEntity = JobExecutionEntity.builder().id(fileDefinition.getJobExecutionId()).build(); jobExecutionEntityRepository.save(jobExecutionEntity); s3Client.write(path, resource.getInputStream()); - inputFileProcessor.readCsvFile(fileDefinition); + inputFileProcessor.readFile(fileDefinition); var total = exportIdEntityRepository.count(); assertEquals(2, total); } @@ -61,6 +62,7 @@ void readCqlFileTest() { var fileDefinition = new FileDefinition(); fileDefinition.setId(UUID.randomUUID()); fileDefinition.fileName("upload.cql"); + fileDefinition.setUploadFormat(FileDefinition.UploadFormatEnum.CQL); fileDefinition.setJobExecutionId(UUID.randomUUID()); var s3Client = folioS3ClientFactory.getFolioS3Client(); @@ -73,7 +75,7 @@ void readCqlFileTest() { var jobExecutionEntity = JobExecutionEntity.builder().id(fileDefinition.getJobExecutionId()).build(); jobExecutionEntityRepository.save(jobExecutionEntity); s3Client.write(path, resource.getInputStream()); - inputFileProcessor.readCqlFile(fileDefinition); + inputFileProcessor.readFile(fileDefinition); var exportIds = exportIdEntityRepository.findAll(); assertEquals(1, exportIds.size()); diff --git a/src/test/java/org/folio/dataexp/service/SingleFileProcessorTest.java b/src/test/java/org/folio/dataexp/service/SingleFileProcessorTest.java new file mode 100644 index 000000000..1e01fd81f --- /dev/null +++ b/src/test/java/org/folio/dataexp/service/SingleFileProcessorTest.java @@ -0,0 +1,58 @@ +package org.folio.dataexp.service; + +import lombok.SneakyThrows; +import org.apache.commons.io.FileUtils; +import org.folio.dataexp.BaseTest; +import org.folio.dataexp.domain.dto.ExportRequest; +import org.folio.dataexp.domain.entity.JobExecutionExportFilesEntity; +import org.folio.dataexp.domain.entity.JobExecutionExportFilesStatus; +import org.folio.dataexp.repository.JobExecutionExportFilesEntityRepository; +import org.folio.dataexp.service.export.storage.FolioS3ClientFactory; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.List; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class SingleFileProcessorTest extends BaseTest { + + @MockBean + private JobExecutionExportFilesEntityRepository jobExecutionExportFilesEntityRepository; + @Autowired + private SingleFileProcessor singleFileProcessor; + @Autowired + private FolioS3ClientFactory folioS3ClientFactory; + + @Test + @SneakyThrows + void exportBySingleFile() { + var s3Client = folioS3ClientFactory.getFolioS3Client(); + + var jobExecutionId = UUID.randomUUID(); + var fileLocation = String.format("mod-data-export/download/%s/download.mrc", jobExecutionId.toString()); + + var exportEntity = JobExecutionExportFilesEntity.builder() + .id(UUID.randomUUID()) + .fileLocation(fileLocation).build(); + + singleFileProcessor.exportBySingleFile(List.of(exportEntity), ExportRequest.RecordTypeEnum.INSTANCE); + + var expectedFile = new File(fileLocation); + var path = Paths.get(fileLocation); + assertTrue(Files.exists(path)); + + long size = s3Client.getSize(fileLocation); + assertTrue(size > 0); + + assertEquals(JobExecutionExportFilesStatus.COMPLETED, exportEntity.getStatus()); + + FileUtils.delete(expectedFile); + } +} diff --git a/src/test/java/org/folio/dataexp/service/SlicerProcessorTest.java b/src/test/java/org/folio/dataexp/service/SlicerProcessorTest.java index 8b6ef211b..02b6197c0 100644 --- a/src/test/java/org/folio/dataexp/service/SlicerProcessorTest.java +++ b/src/test/java/org/folio/dataexp/service/SlicerProcessorTest.java @@ -4,8 +4,9 @@ import org.folio.dataexp.BaseTest; import org.folio.dataexp.domain.dto.FileDefinition; import org.folio.dataexp.domain.entity.JobExecutionEntity; +import org.folio.dataexp.domain.entity.JobExecutionExportFilesStatus; import org.folio.dataexp.repository.JobExecutionEntityRepository; -import org.folio.dataexp.repository.JobExecutionExportFilesRepository; +import org.folio.dataexp.repository.JobExecutionExportFilesEntityRepository; import org.folio.dataexp.service.export.storage.FolioS3ClientFactory; import org.folio.spring.scope.FolioExecutionContextSetter; import org.junit.jupiter.api.Test; @@ -29,7 +30,7 @@ public class SlicerProcessorTest extends BaseTest { private SlicerProcessor slicerProcessor; @Autowired - private JobExecutionExportFilesRepository jobExecutionExportFilesRepository; + private JobExecutionExportFilesEntityRepository jobExecutionExportFilesEntityRepository; @Autowired private JobExecutionEntityRepository jobExecutionEntityRepository; @@ -39,6 +40,7 @@ void sliceInstancesIdsTest() { var fileDefinition = new FileDefinition(); fileDefinition.setId(UUID.randomUUID()); fileDefinition.fileName("upload_for_slicer.cql"); + fileDefinition.setUploadFormat(FileDefinition.UploadFormatEnum.CQL); fileDefinition.setJobExecutionId(UUID.randomUUID()); var s3Client = folioS3ClientFactory.getFolioS3Client(); @@ -51,10 +53,10 @@ void sliceInstancesIdsTest() { var jobExecutionEntity = JobExecutionEntity.builder().id(fileDefinition.getJobExecutionId()).build(); jobExecutionEntityRepository.save(jobExecutionEntity); s3Client.write(path, resource.getInputStream()); - inputFileProcessor.readCqlFile(fileDefinition); + inputFileProcessor.readFile(fileDefinition); slicerProcessor.sliceInstancesIds(fileDefinition, 1); - var exportFiles = jobExecutionExportFilesRepository.findAll(); + var exportFiles = jobExecutionExportFilesEntityRepository.findAll(); assertEquals(2, exportFiles.size()); @@ -62,7 +64,7 @@ void sliceInstancesIdsTest() { var expectedFileLocation = String.format("mod-data-export/download/%s/upload_for_slicer_011e1aea-222d-4d1d-957d-0abcdd0e9acd_011e1aea-222d-4d1d-957d-0abcdd0e9acd.mrc", fileDefinition.getJobExecutionId()); var expectedFromUUID = UUID.fromString("011e1aea-222d-4d1d-957d-0abcdd0e9acd"); var expectedToUUID = UUID.fromString("011e1aea-222d-4d1d-957d-0abcdd0e9acd"); - var expectedStatus = "SCHEDULED"; + var expectedStatus = JobExecutionExportFilesStatus.SCHEDULED; assertEquals(fileDefinition.getJobExecutionId(), joExecutionExportFilesEntity.getJobExecutionId()); assertEquals(expectedFileLocation, joExecutionExportFilesEntity.getFileLocation()); @@ -74,7 +76,7 @@ void sliceInstancesIdsTest() { expectedFileLocation = String.format("mod-data-export/download/%s/upload_for_slicer_011e1aea-111d-4d1d-957d-0abcdd0e9acd_011e1aea-111d-4d1d-957d-0abcdd0e9acd.mrc", fileDefinition.getJobExecutionId()); expectedFromUUID = UUID.fromString("011e1aea-111d-4d1d-957d-0abcdd0e9acd"); expectedToUUID = UUID.fromString("011e1aea-111d-4d1d-957d-0abcdd0e9acd"); - expectedStatus = "SCHEDULED"; + expectedStatus = JobExecutionExportFilesStatus.SCHEDULED; assertEquals(fileDefinition.getJobExecutionId(), joExecutionExportFilesEntity.getJobExecutionId()); assertEquals(expectedFileLocation, joExecutionExportFilesEntity.getFileLocation()); @@ -82,19 +84,19 @@ void sliceInstancesIdsTest() { assertEquals(expectedToUUID, joExecutionExportFilesEntity.getToId()); assertEquals(expectedStatus, joExecutionExportFilesEntity.getStatus()); - jobExecutionExportFilesRepository.deleteAll(); - exportFiles = jobExecutionExportFilesRepository.findAll(); + jobExecutionExportFilesEntityRepository.deleteAll(); + exportFiles = jobExecutionExportFilesEntityRepository.findAll(); assertEquals(0, exportFiles.size()); slicerProcessor.sliceInstancesIds(fileDefinition, 2); - exportFiles = jobExecutionExportFilesRepository.findAll(); + exportFiles = jobExecutionExportFilesEntityRepository.findAll(); assertEquals(1, exportFiles.size()); joExecutionExportFilesEntity = exportFiles.get(0); expectedFileLocation = String.format("mod-data-export/download/%s/upload_for_slicer_011e1aea-222d-4d1d-957d-0abcdd0e9acd_011e1aea-111d-4d1d-957d-0abcdd0e9acd.mrc", fileDefinition.getJobExecutionId()); expectedFromUUID = UUID.fromString("011e1aea-222d-4d1d-957d-0abcdd0e9acd"); expectedToUUID = UUID.fromString("011e1aea-111d-4d1d-957d-0abcdd0e9acd"); - expectedStatus = "SCHEDULED"; + expectedStatus = JobExecutionExportFilesStatus.SCHEDULED; assertEquals(fileDefinition.getJobExecutionId(), joExecutionExportFilesEntity.getJobExecutionId()); assertEquals(expectedFileLocation, joExecutionExportFilesEntity.getFileLocation());