Skip to content

Commit

Permalink
MDEXP-682 Refactored and changed logic
Browse files Browse the repository at this point in the history
  • Loading branch information
@obozhko-folio
obozhko-folio committed Apr 25, 2024
1 parent 4bb2fa6 commit 2e467bc
Show file tree
Hide file tree
Showing 7 changed files with 55 additions and 150 deletions.
20 changes: 1 addition & 19 deletions descriptors/ModuleDescriptor-template.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -429,18 +429,6 @@
],
"modulePermissions": [
]
},
{
"methods": [
"GET"
],
"pathPattern": "/data-export/marc-deleted-ids",
"permissionsRequired": [
"data-export.marc-deleted-ids.collection.get"
],
"modulePermissions": [
"source-storage.records.get"
]
}
]
},
Expand Down Expand Up @@ -620,11 +608,6 @@
"displayName": "Data Export - related users",
"description": "To return only users which executed export"
},
{
"permissionName": "data-export.marc-deleted-ids.collection.get",
"displayName": "Data Export - MARC deleted IDs",
"description": "To return MARC record IDs marked as deleted"
},
{
"permissionName": "data-export.all",
"displayName": "Data Export - all permissions",
Expand Down Expand Up @@ -654,8 +637,7 @@
"data-export.quick.export.post",
"data-export.configuration.post",
"data-export.export.all",
"data-export.related-users.collection.get",
"data-export.marc-deleted-ids.collection.get"
"data-export.related-users.collection.get"
],
"visible": false
}
Expand Down
28 changes: 0 additions & 28 deletions src/main/java/org/folio/dataexp/controllers/MarcDeletedIdsController.java
View file

This file was deleted.

46 changes: 20 additions & 26 deletions src/main/java/org/folio/dataexp/service/MarcDeletedIdsService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,14 @@
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.folio.dataexp.client.SourceStorageClient;
import org.folio.dataexp.domain.dto.MarcDeletedIdsCollection;
import org.folio.dataexp.domain.dto.FileDefinition;
import org.folio.dataexp.domain.dto.MarcRecordIdentifiersPayload;
import org.folio.spring.DefaultFolioExecutionContext;
import org.folio.spring.FolioExecutionContext;
import org.folio.spring.FolioModuleMetadata;
import org.folio.spring.integration.XOkapiHeaders;
import org.folio.spring.scope.FolioExecutionContextSetter;
import org.springframework.core.io.ByteArrayResource;
import org.springframework.stereotype.Service;

import java.util.ArrayList;
Expand All @@ -23,7 +24,6 @@
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.UUID;

import static java.lang.String.format;
import static java.util.Objects.nonNull;
Expand All @@ -43,46 +43,40 @@ public class MarcDeletedIdsService {
private final ConsortiaService consortiaService;
private final FolioExecutionContext folioExecutionContext;
private final FolioModuleMetadata folioModuleMetadata;
private final FileDefinitionsService fileDefinitionsService;

public MarcDeletedIdsCollection getMarcDeletedIds(Date from, Date to, Integer offset, Integer limit) {
public FileDefinition getFileDefinitionForMarcDeletedIds(Date from, Date to) {
var dateFrom = nonNull(from) ? from.toInstant() : null;
var dateTo = nonNull(to) ? to.toInstant() : null;
log.info("GET MARC deleted IDs with date from {}, date to {}, offset {}, limit {}", dateFrom, dateTo, offset,
limit);
var marcDeletedIdsCollection = new MarcDeletedIdsCollection();
var payload = new MarcRecordIdentifiersPayload().withLeaderSearchExpression(LEADER_SEARCH_EXPRESSION_DELETED)
.withOffset(offset).withLimit(limit);
log.info("GET MARC deleted IDs with date from {}, date to {}", dateFrom, dateTo);
var payload = new MarcRecordIdentifiersPayload().withLeaderSearchExpression(LEADER_SEARCH_EXPRESSION_DELETED);
enrichWithDate(payload, from, to);

List<UUID> marcIds = new ArrayList<>();
List<String> marcIds = new ArrayList<>();
marcIds.addAll(fetchFromLocalTenant(payload));
if (nonNull(limit)) {
limit -= marcIds.size();
payload.setLimit(limit);
}
if (nonNull(offset)) {
offset = offset - marcIds.size() < 0 ? 0 : offset - marcIds.size();
payload.setOffset(offset);
}
marcIds.addAll(fetchFromCentralTenant(payload));

marcDeletedIdsCollection.setDeletedMarcIds(marcIds);
marcDeletedIdsCollection.setTotalRecords(marcIds.size());

return marcDeletedIdsCollection;
var fileContent = String.join(System.lineSeparator(), marcIds);

var fileDefinition = new FileDefinition();
fileDefinition.setSize(marcIds.size());
fileDefinition.setUploadFormat(FileDefinition.UploadFormatEnum.CSV);
fileDefinition.setFileName("marcDeletedIds.csv");
fileDefinition = fileDefinitionsService.postFileDefinition(fileDefinition);
fileDefinition = fileDefinitionsService.uploadFile(fileDefinition.getId(), new ByteArrayResource(fileContent.getBytes()));
return fileDefinition;
}

private List<UUID> fetchFromLocalTenant(MarcRecordIdentifiersPayload payload) {
var marcIds = new HashSet<>(sourceStorageClient.getMarcRecordsIdentifiers(payload).getRecords()).stream().map(UUID::fromString).toList();
private List<String> fetchFromLocalTenant(MarcRecordIdentifiersPayload payload) {
var marcIds = new HashSet<>(sourceStorageClient.getMarcRecordsIdentifiers(payload).getRecords()).stream().toList();
log.info("Found deleted MARC IDs from member tenant: {}", marcIds.size());
return marcIds;
}

private List<UUID> fetchFromCentralTenant(MarcRecordIdentifiersPayload payload) {
private List<String> fetchFromCentralTenant(MarcRecordIdentifiersPayload payload) {
var centralTenantId = consortiaService.getCentralTenantId();
if (StringUtils.isNotEmpty(centralTenantId) && !centralTenantId.equals(folioExecutionContext.getTenantId())) {
try (var ignored = new FolioExecutionContextSetter(prepareContextForTenant(centralTenantId, folioModuleMetadata, folioExecutionContext))) {
var marcIdsFromCentral = new HashSet<>(sourceStorageClient.getMarcRecordsIdentifiers(payload).getRecords()).stream().map(UUID::fromString).toList();
var marcIdsFromCentral = new HashSet<>(sourceStorageClient.getMarcRecordsIdentifiers(payload).getRecords()).stream().toList();
log.info("Found deleted MARC IDs from central tenant: {}", marcIdsFromCentral.size());
return marcIdsFromCentral;
}
Expand Down
38 changes: 0 additions & 38 deletions src/main/resources/swagger.api/data-export.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -799,44 +799,6 @@ paths:
content:
text/plain:
example: "Internal server error"
/marc-deleted-ids:
get:
description: Get IDs of MARC records marked as deleted
operationId: getMarcDeletedIds
parameters:
- name: from
in: query
required: false
schema:
type: string
format: date
- name: to
in: query
required: false
schema:
type: string
format: date
- $ref: '#/components/parameters/trait_pageable_offset'
- $ref: '#/components/parameters/trait_pageable_limit'
responses:
'200':
description: IDs of deleted MARCs
content:
application/json:
schema:
$ref: '#/components/schemas/marcDeletedIdsCollection'
example:
$ref: 'examples/marcDeletedIdsCollection.json'
'404':
description: Not Found
content:
text/plain:
example: "Deleted MARC IDs not found"
'500':
description: Internal server errors, e.g. due to misconfiguration
content:
text/plain:
example: "Internal server error"
components:
schemas:
uuid:
Expand Down
6 changes: 0 additions & 6 deletions src/main/resources/swagger.api/examples/marcDeletedIdsCollection.json
View file

This file was deleted.

25 changes: 0 additions & 25 deletions src/main/resources/swagger.api/schemas/marcDeletedIdsCollection.json
View file

This file was deleted.

42 changes: 34 additions & 8 deletions src/test/java/org/folio/dataexp/service/MarcDeletedIdsServiceTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import lombok.SneakyThrows;
import org.folio.dataexp.client.SourceStorageClient;
import org.folio.dataexp.domain.dto.FileDefinition;
import org.folio.dataexp.domain.dto.MarcRecordIdentifiersPayload;
import org.folio.dataexp.domain.dto.MarcRecordsIdentifiersResponse;
import org.folio.spring.FolioExecutionContext;
Expand All @@ -14,6 +15,7 @@
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.core.io.Resource;

import java.text.SimpleDateFormat;
import java.util.List;
Expand All @@ -38,6 +40,8 @@ class MarcDeletedIdsServiceTest {
private FolioExecutionContext folioExecutionContext;
@Mock
private FolioModuleMetadata folioModuleMetadata;
@Mock
private FileDefinitionsService fileDefinitionsService;
@Captor
private ArgumentCaptor<MarcRecordIdentifiersPayload> payloadArgumentCaptor;
@InjectMocks
Expand All @@ -49,9 +53,12 @@ void shouldReturnOneRecord_IfLocalTenantIsCentral() {
.thenReturn(new MarcRecordsIdentifiersResponse().withRecords(List.of(UUID.randomUUID().toString())).withTotalCount(1));
when(consortiaService.getCentralTenantId()).thenReturn("central");
when(folioExecutionContext.getTenantId()).thenReturn("central");
var fileDefinition = new FileDefinition().size(1).id(UUID.randomUUID());
when(fileDefinitionsService.postFileDefinition(isA(FileDefinition.class))).thenReturn(fileDefinition);
when(fileDefinitionsService.uploadFile(isA(UUID.class), isA(Resource.class))).thenReturn(fileDefinition);

var res = marcDeletedIdsService.getMarcDeletedIds(null, null, null, null);
assertThat(res.getTotalRecords()).isEqualTo(1);
var res = marcDeletedIdsService.getFileDefinitionForMarcDeletedIds(null, null);
assertThat(res.getSize()).isEqualTo(1);
}

@Test
Expand All @@ -61,9 +68,12 @@ void shouldReturnTwoRecords_IfLocalTenantIsMember() {
when(consortiaService.getCentralTenantId()).thenReturn("central");
when(folioExecutionContext.getTenantId()).thenReturn("member");
when(folioExecutionContext.getOkapiHeaders()).thenReturn(Map.of(XOkapiHeaders.TENANT, List.of("member")));
var fileDefinition = new FileDefinition().size(2).id(UUID.randomUUID());
when(fileDefinitionsService.postFileDefinition(isA(FileDefinition.class))).thenReturn(fileDefinition);
when(fileDefinitionsService.uploadFile(isA(UUID.class), isA(Resource.class))).thenReturn(fileDefinition);

var res = marcDeletedIdsService.getMarcDeletedIds(null, null, null, null);
assertThat(res.getTotalRecords()).isEqualTo(2);
var res = marcDeletedIdsService.getFileDefinitionForMarcDeletedIds(null, null);
assertThat(res.getSize()).isEqualTo(2);
}

@Test
Expand All @@ -74,8 +84,12 @@ void shouldHavePayloadWithDateRange_IfDateRangeIsUsed() {
when(consortiaService.getCentralTenantId()).thenReturn("central");
when(folioExecutionContext.getTenantId()).thenReturn("member");
when(folioExecutionContext.getOkapiHeaders()).thenReturn(Map.of(XOkapiHeaders.TENANT, List.of("member")));
var fileDefinition = new FileDefinition().id(UUID.randomUUID());
when(fileDefinitionsService.postFileDefinition(isA(FileDefinition.class))).thenReturn(fileDefinition);
when(fileDefinitionsService.uploadFile(isA(UUID.class), isA(Resource.class))).thenReturn(fileDefinition);

var date = new SimpleDateFormat(DATE_PATTERN);
marcDeletedIdsService.getMarcDeletedIds(date.parse("20240424"), date.parse("20240424"), null, null);
marcDeletedIdsService.getFileDefinitionForMarcDeletedIds(date.parse("20240424"), date.parse("20240424"));
verify(sourceStorageClient, times(2)).getMarcRecordsIdentifiers(payloadArgumentCaptor.capture());

var payload = payloadArgumentCaptor.getValue();
Expand All @@ -91,8 +105,12 @@ void shouldHavePayloadWithFromDate_IfDateFromIsUsed() {
when(consortiaService.getCentralTenantId()).thenReturn("central");
when(folioExecutionContext.getTenantId()).thenReturn("member");
when(folioExecutionContext.getOkapiHeaders()).thenReturn(Map.of(XOkapiHeaders.TENANT, List.of("member")));
var fileDefinition = new FileDefinition().id(UUID.randomUUID());
when(fileDefinitionsService.postFileDefinition(isA(FileDefinition.class))).thenReturn(fileDefinition);
when(fileDefinitionsService.uploadFile(isA(UUID.class), isA(Resource.class))).thenReturn(fileDefinition);

var date = new SimpleDateFormat(DATE_PATTERN);
marcDeletedIdsService.getMarcDeletedIds(date.parse("20240424"), null, null, null);
marcDeletedIdsService.getFileDefinitionForMarcDeletedIds(date.parse("20240424"), null);
verify(sourceStorageClient, times(2)).getMarcRecordsIdentifiers(payloadArgumentCaptor.capture());

var payload = payloadArgumentCaptor.getValue();
Expand All @@ -108,8 +126,12 @@ void shouldHavePayloadWithToDate_IfDateToIsUsed() {
when(consortiaService.getCentralTenantId()).thenReturn("central");
when(folioExecutionContext.getTenantId()).thenReturn("member");
when(folioExecutionContext.getOkapiHeaders()).thenReturn(Map.of(XOkapiHeaders.TENANT, List.of("member")));
var fileDefinition = new FileDefinition().id(UUID.randomUUID());
when(fileDefinitionsService.postFileDefinition(isA(FileDefinition.class))).thenReturn(fileDefinition);
when(fileDefinitionsService.uploadFile(isA(UUID.class), isA(Resource.class))).thenReturn(fileDefinition);

var date = new SimpleDateFormat(DATE_PATTERN);
marcDeletedIdsService.getMarcDeletedIds(null, date.parse("20240424"), null, null);
marcDeletedIdsService.getFileDefinitionForMarcDeletedIds(null, date.parse("20240424"));
verify(sourceStorageClient, times(2)).getMarcRecordsIdentifiers(payloadArgumentCaptor.capture());

var payload = payloadArgumentCaptor.getValue();
Expand All @@ -124,7 +146,11 @@ void shouldHavePayloadWithNoDate_IfDateIsNotUsed() {
when(consortiaService.getCentralTenantId()).thenReturn("central");
when(folioExecutionContext.getTenantId()).thenReturn("member");
when(folioExecutionContext.getOkapiHeaders()).thenReturn(Map.of(XOkapiHeaders.TENANT, List.of("member")));
marcDeletedIdsService.getMarcDeletedIds(null, null, null, null);
var fileDefinition = new FileDefinition().id(UUID.randomUUID());
when(fileDefinitionsService.postFileDefinition(isA(FileDefinition.class))).thenReturn(fileDefinition);
when(fileDefinitionsService.uploadFile(isA(UUID.class), isA(Resource.class))).thenReturn(fileDefinition);

marcDeletedIdsService.getFileDefinitionForMarcDeletedIds(null, null);
verify(sourceStorageClient, times(2)).getMarcRecordsIdentifiers(payloadArgumentCaptor.capture());

var payload = payloadArgumentCaptor.getValue();
Expand Down

0 comments on commit 2e467bc

Please sign in to comment.