MDEXP-630 - single file processor

.gitignore

@@ -1,5 +1,6 @@
 HELP.md
 target/
+mod-data-export/
 !.mvn/wrapper/maven-wrapper.jar
 !**/src/main/**/target/
 !**/src/test/**/target/

src/main/java/org/folio/dataexp/controllers/DataExportController.java

@@ -2,23 +2,27 @@
 
 import lombok.RequiredArgsConstructor;
 import lombok.extern.log4j.Log4j2;
+import org.folio.dataexp.domain.dto.ExportRequest;
 import org.folio.dataexp.domain.dto.FileDefinition;
+import org.folio.dataexp.rest.resource.ExportApi;
 import org.folio.dataexp.rest.resource.FileDefinitionsApi;
 import org.folio.dataexp.service.DataExportService;
 import org.springframework.core.io.Resource;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.context.request.NativeWebRequest;
 
+import java.util.Optional;
 import java.util.UUID;
 
 
 @RestController
 @RequiredArgsConstructor
 @Log4j2
 @RequestMapping("/data-export")
-public class DataExportController implements FileDefinitionsApi {
+public class DataExportController implements FileDefinitionsApi, ExportApi {
 
   private final DataExportService dataExportService;
 
@@ -38,4 +42,15 @@ public ResponseEntity<FileDefinition> uploadFile(UUID fileDefinitionId, Resource
     var fileDefinition = dataExportService.uploadFile(fileDefinitionId, resource);
     return new ResponseEntity<>(fileDefinition, HttpStatus.OK);
   }
+
+  @Override
+  public ResponseEntity<Void> postDataExport(ExportRequest exportRequest) {
+    dataExportService.postDataExport(exportRequest);
+    return new ResponseEntity<>(HttpStatus.OK);
+  }
+
+  @Override
+  public Optional<NativeWebRequest> getRequest() {
+    return Optional.empty();
+  }
 }

src/main/java/org/folio/dataexp/domain/entity/JobExecutionEntity.java

@@ -7,13 +7,15 @@
 import jakarta.persistence.Table;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
+import lombok.Data;
 import lombok.NoArgsConstructor;
 import lombok.With;
 import org.folio.dataexp.domain.dto.JobExecution;
 import org.hibernate.annotations.Type;
 
 import java.util.UUID;
 
+@Data
 @Builder
 @With
 @AllArgsConstructor

src/main/java/org/folio/dataexp/domain/entity/JobExecutionExportFilesEntity.java

@@ -1,6 +1,8 @@
 package org.folio.dataexp.domain.entity;
 
 import jakarta.persistence.Entity;
+import jakarta.persistence.EnumType;
+import jakarta.persistence.Enumerated;
 import jakarta.persistence.Id;
 import jakarta.persistence.Table;
 import lombok.AllArgsConstructor;
@@ -31,5 +33,6 @@ public class JobExecutionExportFilesEntity {
 
   private UUID toId;
 
-  private String status;
+  @Enumerated(EnumType.STRING)
+  private JobExecutionExportFilesStatus status;
 }

src/main/java/org/folio/dataexp/domain/entity/JobExecutionExportFilesStatus.java

@@ -0,0 +1,5 @@
+package org.folio.dataexp.domain.entity;
+
+public enum JobExecutionExportFilesStatus {
+  SCHEDULED, ACTIVE, COMPLETED, FAILED;
+}

src/main/java/org/folio/dataexp/exception/DataExportExceptionHandler.java

@@ -1,6 +1,7 @@
 package org.folio.dataexp.exception;
 
 import jakarta.persistence.EntityNotFoundException;
+import org.folio.dataexp.exception.export.DataExportException;
 import org.folio.dataexp.exception.export.FileExtensionException;
 import org.folio.dataexp.exception.export.FileSizeException;
 import org.folio.dataexp.exception.export.UploadFileException;
@@ -31,4 +32,9 @@ public ResponseEntity<String> handleUploadFileException(final UploadFileExceptio
   public ResponseEntity<String> handleEntityNotFoundException(final EntityNotFoundException e) {
     return new ResponseEntity<>(e.getMessage(), HttpStatus.NOT_FOUND);
   }
+
+  @ExceptionHandler(DataExportException.class)
+  public ResponseEntity<String> handleDataExportException(final DataExportException e) {
+    return new ResponseEntity<>(e.getMessage(), HttpStatus.INTERNAL_SERVER_ERROR);
+  }
 }

src/main/java/org/folio/dataexp/exception/export/DataExportException.java

@@ -0,0 +1,7 @@
+package org.folio.dataexp.exception.export;
+
+public class DataExportException extends RuntimeException{
+  public DataExportException(String message) {
+    super(message);
+  }
+}

src/main/java/org/folio/dataexp/repository/JobExecutionExportFilesEntityRepository.java

@@ -0,0 +1,12 @@
+package org.folio.dataexp.repository;
+
+import org.folio.dataexp.domain.entity.JobExecutionExportFilesEntity;
+import org.springframework.data.jpa.repository.JpaRepository;
+
+import java.util.List;
+import java.util.UUID;
+
+public interface JobExecutionExportFilesEntityRepository extends JpaRepository<JobExecutionExportFilesEntity, UUID> {
+
+  List<JobExecutionExportFilesEntity> findByJobExecutionId(UUID jobExecutionId);
+}

src/main/java/org/folio/dataexp/service/DataExportService.java

@@ -2,55 +2,44 @@
 
 import lombok.RequiredArgsConstructor;
 import lombok.extern.log4j.Log4j2;
-import org.apache.commons.io.FilenameUtils;
+import org.folio.dataexp.domain.dto.ExportRequest;
 import org.folio.dataexp.domain.dto.FileDefinition;
 import org.folio.dataexp.domain.dto.JobExecution;
 import org.folio.dataexp.domain.entity.FileDefinitionEntity;
 import org.folio.dataexp.domain.entity.JobExecutionEntity;
-import org.folio.dataexp.exception.export.FileExtensionException;
-import org.folio.dataexp.exception.export.FileSizeException;
+import org.folio.dataexp.exception.export.DataExportException;
 import org.folio.dataexp.exception.export.UploadFileException;
 import org.folio.dataexp.repository.FileDefinitionEntityRepository;
 import org.folio.dataexp.repository.JobExecutionEntityRepository;
+import org.folio.dataexp.repository.JobExecutionExportFilesEntityRepository;
+import org.folio.dataexp.repository.JobProfileEntityRepository;
 import org.folio.dataexp.service.file.upload.FileUploadService;
 import org.folio.spring.FolioExecutionContext;
 import org.springframework.core.io.Resource;
 import org.springframework.stereotype.Service;
 
 import java.time.LocalDateTime;
-import java.util.Objects;
 import java.util.UUID;
 
 @Service
 @RequiredArgsConstructor
 @Log4j2
 public class DataExportService {
 
-  private static final String CSV_FORMAT_EXTENSION = "csv";
-  private static final String CQL_FORMAT_EXTENSION = "cql";
-
-  private static final int MAX_FILE_SIZE = 500_000;
-
   private final FileDefinitionEntityRepository fileDefinitionEntityRepository;
   private final JobExecutionEntityRepository jobExecutionEntityRepository;
+  private final JobProfileEntityRepository jobProfileEntityRepository;
+  private final JobExecutionExportFilesEntityRepository jobExecutionExportFilesEntityRepository;
   private final FileUploadService fileUploadService;
+  private final InputFileProcessor inputFileProcessor;
+  private final SlicerProcessor slicerProcessor;
+  private final SingleFileProcessor singleFileProcessor;
+  private final FileDefinitionValidator fileDefinitionValidator;
   private final FolioExecutionContext folioExecutionContext;
 
   public FileDefinition postFileDefinition(FileDefinition fileDefinition) {
     log.info("Post file definition by id {}", fileDefinition.getId());
-    if (Objects.nonNull(fileDefinition.getSize()) && fileDefinition.getSize() > MAX_FILE_SIZE) {
-      var errorMessage = String.format("File size is too large: '%d'. Please use file with size less than %d.", fileDefinition.getSize(), MAX_FILE_SIZE);
-      log.error(errorMessage);
-      throw new FileSizeException(errorMessage);
-    }
-    if (Objects.isNull(fileDefinition.getSize())) {
-      log.error("Size of uploading file is null.");
-    }
-    if (isNotValidFileNameExtension(fileDefinition.getFileName())) {
-      var errorMessage = String.format("Incorrect file extension of %s", fileDefinition.getFileName());
-      log.error(errorMessage);
-      throw new FileExtensionException(errorMessage);
-    }
+    fileDefinitionValidator.validate(fileDefinition);
     var jobExecution = new JobExecution();
     jobExecution.setId(UUID.randomUUID());
     jobExecutionEntityRepository.save(JobExecutionEntity.builder()
@@ -79,7 +68,23 @@ public FileDefinition uploadFile(UUID fileDefinitionId, Resource resource) {
     }
   }
 
-  private boolean isNotValidFileNameExtension(String fileName) {
-    return !FilenameUtils.isExtension(fileName.toLowerCase(), CSV_FORMAT_EXTENSION) && !FilenameUtils.isExtension(fileName.toLowerCase(), CQL_FORMAT_EXTENSION);
+  public void postDataExport(ExportRequest exportRequest) {
+    log.info("Post data export for file definition {} and job profile {}", exportRequest.getFileDefinitionId(), exportRequest.getJobProfileId());
+    var fileDefinition = fileDefinitionEntityRepository.
+      getReferenceById(exportRequest.getFileDefinitionId()).getFileDefinition();
+    var jobProfileEntity = jobProfileEntityRepository.getReferenceById(exportRequest.getJobProfileId());
+    var jobExecutionEntity = jobExecutionEntityRepository.getReferenceById(fileDefinition.getJobExecutionId());
+    var jobExecution = jobExecutionEntity.getJobExecution();
+    jobExecution.setJobProfileId(jobProfileEntity.getJobProfile().getId());
+    jobExecution.setJobProfileName(jobProfileEntity.getJobProfile().getName());
+    jobExecutionEntityRepository.save(jobExecutionEntity);
+    try {
+      inputFileProcessor.readFile(fileDefinition);
+      slicerProcessor.sliceInstancesIds(fileDefinition);
+     } catch (Exception e) {
+      throw new DataExportException(e.getMessage());
+    }
+    var jobExecutionExportFilesEntities = jobExecutionExportFilesEntityRepository.findByJobExecutionId(jobExecution.getId());
+    singleFileProcessor.exportBySingleFile(jobExecutionExportFilesEntities, exportRequest.getRecordType());
   }
 }

src/main/java/org/folio/dataexp/service/FileDefinitionValidator.java

@@ -0,0 +1,40 @@
+package org.folio.dataexp.service;
+
+import lombok.extern.log4j.Log4j2;
+import org.apache.commons.io.FilenameUtils;
+import org.folio.dataexp.domain.dto.FileDefinition;
+import org.folio.dataexp.exception.export.FileExtensionException;
+import org.folio.dataexp.exception.export.FileSizeException;
+import org.springframework.stereotype.Component;
+
+import java.util.Objects;
+
+@Component
+@Log4j2
+public class FileDefinitionValidator {
+
+  private static final String CSV_FORMAT_EXTENSION = "csv";
+  private static final String CQL_FORMAT_EXTENSION = "cql";
+
+  private static final int MAX_FILE_SIZE = 500_000;
+
+  public void validate(FileDefinition fileDefinition) {
+    if (Objects.nonNull(fileDefinition.getSize()) && fileDefinition.getSize() > MAX_FILE_SIZE) {
+      var errorMessage = String.format("File size is too large: '%d'. Please use file with size less than %d.", fileDefinition.getSize(), MAX_FILE_SIZE);
+      log.error(errorMessage);
+      throw new FileSizeException(errorMessage);
+    }
+    if (Objects.isNull(fileDefinition.getSize())) {
+      log.error("Size of uploading file is null.");
+    }
+    if (isNotValidFileNameExtension(fileDefinition.getFileName())) {
+      var errorMessage = String.format("Incorrect file extension of %s", fileDefinition.getFileName());
+      log.error(errorMessage);
+      throw new FileExtensionException(errorMessage);
+    }
+  }
+
+  private boolean isNotValidFileNameExtension(String fileName) {
+    return !FilenameUtils.isExtension(fileName.toLowerCase(), CSV_FORMAT_EXTENSION) && !FilenameUtils.isExtension(fileName.toLowerCase(), CQL_FORMAT_EXTENSION);
+  }
+}

src/main/java/org/folio/dataexp/service/InputFileProcessor.java

@@ -40,7 +40,15 @@ public class InputFileProcessor {
   private final FolioS3ClientFactory folioS3ClientFactory;
   private final JdbcTemplate jdbcTemplate;
 
-  public void readCsvFile(FileDefinition fileDefinition) throws IOException {
+  public void readFile(FileDefinition fileDefinition) throws Exception {
+    if (fileDefinition.getUploadFormat() == FileDefinition.UploadFormatEnum.CQL) {
+      readCqlFile(fileDefinition);
+      return;
+    }
+    readCsvFile(fileDefinition);
+  }
+
+  private void readCsvFile(FileDefinition fileDefinition) throws IOException {
     var pathToRead = getPathToRead(fileDefinition);
     var s3Client = folioS3ClientFactory.getFolioS3Client();
     try (InputStream is = s3Client.read(pathToRead); BufferedReader reader = new BufferedReader(new InputStreamReader(is))) {
@@ -53,7 +61,7 @@ public void readCsvFile(FileDefinition fileDefinition) throws IOException {
     }
   }
 
-  public void readCqlFile(FileDefinition fileDefinition) throws IOException, ServerChoiceIndexesException, FieldException, QueryValidationException, SQLException {
+  private void readCqlFile(FileDefinition fileDefinition) throws IOException, ServerChoiceIndexesException, FieldException, QueryValidationException, SQLException {
     var pathToRead = getPathToRead(fileDefinition);
     var s3Client = folioS3ClientFactory.getFolioS3Client();
     String cql;

src/main/java/org/folio/dataexp/service/SingleFileProcessor.java

@@ -0,0 +1,61 @@
+package org.folio.dataexp.service;
+
+import lombok.RequiredArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+import org.folio.dataexp.domain.dto.ExportRequest;
+import org.folio.dataexp.domain.entity.JobExecutionExportFilesEntity;
+import org.folio.dataexp.domain.entity.JobExecutionExportFilesStatus;
+import org.folio.dataexp.repository.JobExecutionExportFilesEntityRepository;
+import org.folio.dataexp.service.export.storage.FolioS3ClientFactory;
+import org.folio.dataexp.service.export.strategies.ExportStrategy;
+import org.springframework.stereotype.Component;
+
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+
+@Component
+@RequiredArgsConstructor
+@Log4j2
+public class SingleFileProcessor {
+
+  private final ExportStrategy instanceExportStrategy;
+  private final FolioS3ClientFactory folioS3ClientFactory;
+  private final JobExecutionExportFilesEntityRepository jobExecutionExportFilesEntityRepository;
+
+  public void exportBySingleFile(List<JobExecutionExportFilesEntity> exports, ExportRequest.RecordTypeEnum recordType) {
+    exports.forEach(export -> {
+      try {
+        createExportFile(export, recordType);
+      } catch (IOException e) {
+        export.setStatus(JobExecutionExportFilesStatus.FAILED);
+        jobExecutionExportFilesEntityRepository.save(export);
+        log.error("Error exporting {} for jobExecution {}", export.getFileLocation(), export.getJobExecutionId());
+      }
+    });
+  }
+
+  private void createExportFile(JobExecutionExportFilesEntity exportFilesEntity, ExportRequest.RecordTypeEnum recordType) throws IOException {
+    exportFilesEntity.setStatus(JobExecutionExportFilesStatus.ACTIVE);
+    jobExecutionExportFilesEntityRepository.save(exportFilesEntity);
+    var file = new File(exportFilesEntity.getFileLocation());
+    var parent = file.getParentFile();
+    if (!parent.exists()) {
+      boolean isCreated = parent.mkdirs();
+      if (isCreated) log.info("Create local directories to store {} ", exportFilesEntity.getFileLocation());
+    }
+    boolean isCreated = file.createNewFile();
+    if (isCreated) log.info("Create {} locally", exportFilesEntity.getFileLocation());
+    instanceExportStrategy.saveMarc(exportFilesEntity, file);
+    var s3Client = folioS3ClientFactory.getFolioS3Client();
+    try (InputStream is = new BufferedInputStream(new FileInputStream(file))) {
+      s3Client.write(exportFilesEntity.getFileLocation(), is);
+      log.info("Create {} at remote storage", exportFilesEntity.getFileLocation());
+    }
+    exportFilesEntity.setStatus(JobExecutionExportFilesStatus.COMPLETED);
+    jobExecutionExportFilesEntityRepository.save(exportFilesEntity);
+  }
+}

src/main/java/org/folio/dataexp/service/SlicerProcessor.java

@@ -15,6 +15,7 @@
 @RequiredArgsConstructor
 @Log4j2
 public class SlicerProcessor {
+
   private static final String CALL_SLICE_INSTANCES_IDS_PROCEDURE = "call slice_instances_ids(?, ?, ?)";
   private static final String SLICED_FILE_LOCATION_PATH = "mod-data-export/download/%s/";
   private static final String FROM_TO_UUID_PART = "_%s_%s";

src/main/java/org/folio/dataexp/service/export/strategies/ExportStrategy.java

@@ -0,0 +1,11 @@
+package org.folio.dataexp.service.export.strategies;
+
+import org.folio.dataexp.domain.entity.JobExecutionExportFilesEntity;
+
+import java.io.File;
+
+public interface ExportStrategy {
+
+  void saveMarc(JobExecutionExportFilesEntity exportFilesEntity, File file);
+
+}