Skip to content

Commit

Permalink
Merge pull request #84 from aodn/features/5694-upgrade-test-container
Browse files Browse the repository at this point in the history 
Features/5694 upgrade test container
  • Loading branch information
@vietnguyengit
vietnguyengit authored Jul 5, 2024
2 parents 5d0481a + 4cee523 commit 5d20dca
Show file tree
Hide file tree
Showing 8 changed files with 195 additions and 154 deletions.
34 changes: 30 additions & 4 deletions indexer/src/main/java/au/org/aodn/esindexer/configuration/GeoNetworkSearchConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import au.org.aodn.esindexer.utils.UrlUtils;
import co.elastic.clients.elasticsearch.ElasticsearchClient;
import co.elastic.clients.json.jackson.JacksonJsonpMapper;
import co.elastic.clients.transport.ElasticsearchTransportBase;
import co.elastic.clients.transport.rest_client.RestClientTransport;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpHost;
Expand All @@ -17,16 +18,40 @@
import org.springframework.http.MediaType;
import org.springframework.web.client.RestTemplate;

import java.lang.reflect.Field;
import java.util.Set;

@Configuration
public class GeoNetworkSearchConfig {

@Bean(name = "gn4ElasticsearchClient")
public ElasticsearchClient createGN4ElasticsearchClient(@Qualifier("gn4ElasticRestClient") RestClient restClient) {
public ElasticsearchClient createGN4ElasticsearchClient(@Qualifier("gn4ElasticRestClient") RestClient restClient) throws NoSuchFieldException, IllegalAccessException {

RestClientTransport c = new RestClientTransport(restClient, new JacksonJsonpMapper());
/*
TODO: You may need to revisit this setup when elastic-java-client upgrade
Fix due to elastic search api update
Need a hack to work around an issue with geonetwork, geonetwork exposed the ElasticSearch api via endpoint _search
however this is a proxy to the underlying api and unfortunately the proxy do not populate all the header back, namely the
"X-Elastic-Product", which is need in the elastic client api check. This will cause the fail check in the
ElasticTransportBase.checkProductHeader.
To workaround it you need to set the endpointsMsssingProductHeader with value "es/search"
*/
Field endpointsMissingProductHeader = ElasticsearchTransportBase.class.getDeclaredField("endpointsMissingProductHeader");

endpointsMissingProductHeader.setAccessible(true);
Set<String> v = (Set<String>)endpointsMissingProductHeader.get(c);
v.add("es/search");

// Create the API client
return new ElasticsearchClient(c);
// Create the API client, the transport options is needed because the header from the default elastic client
// call is application/vnd.elasticsearch.... which is not something the geonetwork wants. So override it here
// otherwise the function call will always fail due to header value.
return new ElasticsearchClient(c)
.withTransportOptions(
f -> f.setHeader(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE)
.setHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE)
);
}

@Bean(name = "gn4ElasticRestClient")
Expand All @@ -38,7 +63,8 @@ public RestClient createRestClientTransport1(@Value("${geonetwork.host}") String
.builder(HttpHost.create(host))
.setPathPrefix(path)
.setDefaultHeaders(new BasicHeader[] {
new BasicHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE)
new BasicHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE),
new BasicHeader(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE)
})
.build();
}
Expand Down
12 changes: 7 additions & 5 deletions indexer/src/main/java/au/org/aodn/esindexer/service/GeoNetworkServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@

import co.elastic.clients.elasticsearch.core.SearchRequest;
import co.elastic.clients.elasticsearch.core.SearchResponse;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.logging.log4j.LogManager;
Expand Down Expand Up @@ -49,7 +48,7 @@ public class GeoNetworkServiceImpl implements GeoNetworkService {
@Autowired
protected GeoNetworkServiceImpl self;

@Value("${elasticsearch.query.pageSize:1500}")
@Value("${elasticsearch.query.pageSize:500}")
protected int ES_PAGE_SIZE;

protected static final Logger logger = LogManager.getLogger(GeoNetworkServiceImpl.class);
Expand Down Expand Up @@ -448,11 +447,11 @@ public boolean isMetadataRecordsCountLessThan(int c) {
)
@Override
public Iterable<String> getAllMetadataRecords() {

SearchRequest req = createSearchAllUUID(null);
try {
final AtomicReference<String> lastUUID = new AtomicReference<>(null);
final AtomicReference<SearchResponse<ObjectNode>> response =
new AtomicReference<>(gn4ElasticClient.search(createSearchAllUUID(null), ObjectNode.class));
new AtomicReference<>(gn4ElasticClient.search(req, ObjectNode.class));

if(response.get().hits() != null
&& response.get().hits().hits() != null
Expand Down Expand Up @@ -527,7 +526,10 @@ private String getUUID(int index) {
}
}
catch(IOException e) {
throw new RuntimeException("Failed to fetch data from GeoNetwork Elastic API, too busy?");
throw new RuntimeException(
String.format("Failed to fetch data from GeoNetwork Elastic API, too busy? Query is %s", req.toString()),
e
);
}
}

Expand Down
177 changes: 83 additions & 94 deletions indexer/src/main/java/au/org/aodn/esindexer/utils/MapperUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,23 +34,24 @@ public static class Contacts {
}

public static List<String> mapContactsRole(CIResponsibilityType2 ciResponsibility) {
var contactsRole = safeGet(() -> ciResponsibility.getRole().getCIRoleCode().getCodeListValue());
return contactsRole == null ? Collections.emptyList() : Collections.singletonList(contactsRole);
return safeGet(() -> ciResponsibility.getRole().getCIRoleCode().getCodeListValue())
.map(Collections::singletonList)
.orElse(Collections.emptyList());
}

public static String mapContactsOrganization(AbstractCIPartyPropertyType2 party) {
var orgString = safeGet(() -> party.getAbstractCIParty().getValue().getName().getCharacterString().getValue().toString());
return orgString == null? "" : orgString;
return safeGet(() -> party.getAbstractCIParty().getValue().getName().getCharacterString().getValue().toString())
.orElse("");
}

public static String mapContactsName(CIIndividualPropertyType2 individual) {
var name = safeGet(() -> individual.getCIIndividual().getName().getCharacterString().getValue().toString());
return name == null ? "" : name;
return safeGet(() -> individual.getCIIndividual().getName().getCharacterString().getValue().toString())
.orElse("");
}

public static String mapContactsPosition(CIIndividualPropertyType2 individual) {
var position = safeGet(() -> individual.getCIIndividual().getPositionName().getCharacterString().getValue().toString());
return position == null ? "" : position;
return safeGet(() -> individual.getCIIndividual().getPositionName().getCharacterString().getValue().toString())
.orElse("");
}
/**
* Attribute will not be there if it is empty, this align with what Elastic handle null or empty field.
Expand All @@ -59,83 +60,67 @@ public static String mapContactsPosition(CIIndividualPropertyType2 individual) {
*/
public static ContactsAddressModel mapContactsAddress(CIAddressPropertyType2 address) {
ContactsAddressModel addressItem = ContactsAddressModel.builder().build();
List<String> deliveryPoints = new ArrayList<>();

address.getCIAddress().getDeliveryPoint().forEach(deliveryPoint -> {
String deliveryPointString = deliveryPoint.getCharacterString().getValue().toString();
deliveryPoints.add(deliveryPointString != null ? deliveryPointString : "");
});
safeGet(() -> address.getCIAddress().getDeliveryPoint()).ifPresent((value -> {
List<String> deliveryPoints = new ArrayList<>();

if (!deliveryPoints.isEmpty()) {
addressItem.setDeliveryPoint(deliveryPoints);
}
value.forEach(deliveryPoint -> {
String deliveryPointString = deliveryPoint.getCharacterString().getValue().toString();
deliveryPoints.add(deliveryPointString != null ? deliveryPointString : "");
});

var city = safeGet(() -> address.getCIAddress().getCity().getCharacterString().getValue().toString());
if (city != null) {
addressItem.setCity(city);
}
var administrativeArea = safeGet(() -> address.getCIAddress().getAdministrativeArea().getCharacterString().getValue().toString());
if (administrativeArea != null) {
addressItem.setAdministrativeArea(administrativeArea);
}
var postalCode = safeGet(() -> address.getCIAddress().getPostalCode().getCharacterString().getValue().toString());
if (postalCode != null) {
addressItem.setPostalCode(postalCode);
}
var country = safeGet(() -> address.getCIAddress().getCountry().getCharacterString().getValue().toString());
if (country != null) {
addressItem.setCountry(country);
}
if (!deliveryPoints.isEmpty()) {
addressItem.setDeliveryPoint(deliveryPoints);
}
}));

return addressItem;
}
safeGet(() -> address.getCIAddress().getCity().getCharacterString().getValue().toString())
.ifPresent(addressItem::setCity);

public static String mapContactsEmail(CharacterStringPropertyType electronicMailAddress) {
var email = safeGet(() -> electronicMailAddress.getCharacterString().getValue().toString());
safeGet(() -> address.getCIAddress().getAdministrativeArea().getCharacterString().getValue().toString())
.ifPresent(addressItem::setAdministrativeArea);

if (email == null || email.trim().isEmpty()) {
return null;
}
return email;
}
safeGet(() -> address.getCIAddress().getPostalCode().getCharacterString().getValue().toString())
.ifPresent(addressItem::setPostalCode);

safeGet(() -> address.getCIAddress().getCountry().getCharacterString().getValue().toString())
.ifPresent(addressItem::setCountry);

public static String mapContactsEmail2(CharacterStringPropertyType electronicMailAddress) {
var email = safeGet(() -> electronicMailAddress.getCharacterString().getValue().toString());
return addressItem;
}

public static String mapContactsEmail(CharacterStringPropertyType electronicMailAddress) {
Optional<String> email = safeGet(() -> electronicMailAddress.getCharacterString().getValue().toString());

if (email != null && !email.trim().isEmpty()) {
return email;
if (email.isPresent() && !email.get().trim().isEmpty()) {
return email.get();
}
return null;
}

public static LinkModel mapContactsOnlineResource(CIOnlineResourcePropertyType2 onlineResource) {
LinkModel onlineResourceItem = LinkModel.builder().build();

onlineResourceItem.setHref(safeGet(() ->
onlineResource.getCIOnlineResource().getLinkage().getCharacterString().getValue().toString()));
safeGet(() -> onlineResource.getCIOnlineResource().getLinkage().getCharacterString().getValue().toString())
.ifPresent(onlineResourceItem::setHref);

onlineResourceItem.setTitle(safeGet(() ->
onlineResource.getCIOnlineResource().getName().getCharacterString().getValue().toString()));
safeGet(() -> onlineResource.getCIOnlineResource().getName().getCharacterString().getValue().toString())
.ifPresent(onlineResourceItem::setTitle);

onlineResourceItem.setType(safeGet(() ->
onlineResource.getCIOnlineResource().getProtocol().getCharacterString().getValue().toString()));
safeGet(() -> onlineResource.getCIOnlineResource().getProtocol().getCharacterString().getValue().toString())
.ifPresent(onlineResourceItem::setType);

return onlineResourceItem;
}

public static ContactsPhoneModel mapContactsPhone(CITelephonePropertyType2 phone) {
ContactsPhoneModel phoneItem = ContactsPhoneModel.builder().build();

String phoneStr = safeGet(() -> phone.getCITelephone().getNumber().getCharacterString().getValue().toString());
if (phoneStr != null) {
phoneItem.setValue(phoneStr);
}
safeGet(() -> phone.getCITelephone().getNumber().getCharacterString().getValue().toString())
.ifPresent(phoneItem::setValue);

var roleStr = safeGet(() -> phone.getCITelephone().getNumberType().getCITelephoneTypeCode().getCodeListValue());
if (roleStr != null && !roleStr.isEmpty()) {
phoneItem.setRoles(List.of(roleStr));
}
safeGet(() -> phone.getCITelephone().getNumberType().getCITelephoneTypeCode().getCodeListValue())
.ifPresent(roleStr -> phoneItem.setRoles(List.of(roleStr)));

return phoneItem;
}
Expand Down Expand Up @@ -355,38 +340,40 @@ public static Optional<Contacts> mapContactInfo(List<CIContactPropertyType2> con
contactsProperty.forEach(contact -> {

// Add all address of organization
var addresses = safeGet(() -> contact.getCIContact().getAddress());
if (addresses != null) {
addresses.forEach(address -> {

ContactsAddressModel addressModel = mapContactsAddress(address);
if (addressModel.isEmpty()) {
return;
}
contacts.getAddresses().add(addressModel);

var electronicMailAddress = safeGet(() -> address.getCIAddress().getElectronicMailAddress());
if (electronicMailAddress != null) {
contacts.getEmails().addAll(
electronicMailAddress
.stream()
.map(MapperUtils::mapContactsEmail)
.filter(Objects::nonNull)
.toList());
}
});
}
safeGet(() -> contact.getCIContact().getAddress())
.ifPresent(addresses -> {
addresses.forEach(address -> {
ContactsAddressModel addressModel = mapContactsAddress(address);
if (addressModel.isEmpty()) {
return;
}
contacts.getAddresses().add(addressModel);

safeGet(() -> address.getCIAddress().getElectronicMailAddress())
.ifPresent(electronicMailAddress ->
contacts.getEmails().addAll(
electronicMailAddress
.stream()
.map(MapperUtils::mapContactsEmail)
.filter(Objects::nonNull)
.toList())
);
});
});

// Add phone number of organization
var phone = safeGet(() -> contact.getCIContact().getPhone());
if (phone != null) {
contacts.getPhones().addAll(phone.stream().map(MapperUtils::mapContactsPhone).toList());
}
safeGet(() -> contact.getCIContact().getPhone())
.ifPresent(phone ->
contacts.getPhones()
.addAll(phone.stream().map(MapperUtils::mapContactsPhone).toList())
);

// Online resources
var onlineResource = safeGet(() -> contact.getCIContact().getOnlineResource());
if (onlineResource != null) {
contacts.getOnlineResources().addAll(onlineResource.stream().map(MapperUtils::mapContactsOnlineResource).toList());
}
safeGet(() -> contact.getCIContact().getOnlineResource())
.ifPresent(onlineResource ->
contacts.getOnlineResources()
.addAll(onlineResource.stream().map(MapperUtils::mapContactsOnlineResource).toList())
);
});
return Optional.of(contacts);
}
Expand All @@ -395,7 +382,8 @@ public static Optional<Contacts> mapContactInfo(List<CIContactPropertyType2> con
public static List<ContactsModel> mapContactsFromOrg(CIResponsibilityType2 ciResponsibility, CIOrganisationType2 organisation) {

Optional<Contacts> org = mapContactInfo(organisation.getContactInfo());
if (safeGet(organisation::getIndividual) == null) {

if (safeGet(organisation::getIndividual).isEmpty()) {
return Collections.emptyList();
}
return new ArrayList<>(organisation
Expand All @@ -406,8 +394,9 @@ public static List<ContactsModel> mapContactsFromOrg(CIResponsibilityType2 ciRes
contactsModel.setName(mapContactsName(individual));
contactsModel.setPosition(mapContactsPosition(individual));
contactsModel.setRoles(mapContactsRole(ciResponsibility));
var orgName = safeGet(() -> organisation.getName().getCharacterString().getValue().toString());
contactsModel.setOrganization(orgName == null ? "" : orgName);

safeGet(() -> organisation.getName().getCharacterString().getValue().toString())
.ifPresentOrElse(contactsModel::setOrganization, () -> contactsModel.setOrganization(""));

Optional<Contacts> individualContacts = mapContactInfo(individual.getCIIndividual().getContactInfo());
Contacts orgContacts = org.orElse(null);
Expand Down Expand Up @@ -494,11 +483,11 @@ public static List<ContactsModel> addRoleToContacts(List<ContactsModel> contacts
* @param <T> The type of the return value
* @return null if any of the getter is null
*/
public static <T> T safeGet(Supplier<T> supplier) {
public static <T> Optional<T> safeGet(Supplier<T> supplier) {
try {
return supplier.get();
return Optional.of(supplier.get());
} catch (NullPointerException ignored) {
return null;
return Optional.empty();
}
}
}
Loading

0 comments on commit 5d20dca

Please sign in to comment.