Merge pull request #1 from SevKohler/performance_improvements

Performance improvements

README.md

@@ -10,7 +10,7 @@ Eos
 * Postgres if not using docker
 * optionally: openEHR platform, otherwise there is the option to setup one down below
 
-### Setting up an OMOP Db and connect it to an openEHR platform
+### Setting up Eos
 
 1. clone repo `git clone --recursive`
 2. Use Common Data model sqls from `setup/ddls/` or download
@@ -83,11 +83,7 @@ $ mvn clean install
      url: jdbc:postgresql://localhost:5433/YOUR_SCHEMA #default is public
 ```
 
-3. configure openEHR platform in `src/main/resources/application.yml`. <br>
-   If you do not have setup an openEHR platform you can use [ehrbase](https://github.com/ehrbase/ehrbase).
-   (
-   See [here](https://ehrbase.readthedocs.io/en/latest/03_development/04_docker_images/01_ehrbase/02_use_image/index.html#run-ehrbase-db-with-docker-compose)
-   for a guide how to set it up with docker-compose.)
+3. configure openEHR platform in `src/main/resources/application.yml`.
 
 ```yml
   ehrbase:
@@ -101,6 +97,9 @@ $ mvn clean install
 
 4. configure cronjob for eras or leave them for manual execution
 
+
+5. **POSTMAN** examples can be found in `.config/`
+
 ## Mappings
 
 We currently support CDM v5.4, if more tooling is provided v6 will be added in the future.

setup/ddls/create_sequences_and_ehr_to_person.sql

@@ -31,3 +31,4 @@ CREATE TABLE IF NOT EXISTS @cdmDatabaseSchema.ehr_to_person
 
 ALTER TABLE IF EXISTS @cdmDatabaseSchema.ehr_to_person
     OWNER to postgres;
+

src/main/java/org/bih/eos/controller/EhrController.java

@@ -37,7 +37,7 @@ public ResponseEntity<Object> convertAll(@RequestBody String json) {
         ObjectMapper objectMapper = new ObjectMapper();
         try {
             Ehrs ehrs = objectMapper.readValue(json, Ehrs.class);
-            return new ResponseEntity<>(new EhrService(ehrToPersonService, openEhrClient, converterService).convertEhrList(ehrs), HttpStatus.OK);
+            return new ResponseEntity<>(new EhrService(ehrToPersonService, openEhrClient, converterService).convertSpecific(ehrs), HttpStatus.OK);
         } catch (JsonProcessingException e) {
             return new ResponseEntity<>("{ \"message\" : \"Json malformed\" }", HttpStatus.BAD_REQUEST);
         }

src/main/java/org/bih/eos/converter/cdm_field/CDMConverter.java

@@ -29,8 +29,11 @@ private Optional<T> runConversion(Locatable contentItem, ValueEntry[] valueEntri
         for (ValueEntry valueEntry : valueEntries) {
             if (valueEntry.getCode() != null) {
                 return convertCode(valueEntry.getCode());
-            } else if (valueEntry.getPath() != null && PathProcessor.getItemAtPath(contentItem, valueEntry.getPath()).isPresent() && convertPath(contentItem, valueEntry.getPath()).isPresent() ) {
-                return convertPath(contentItem, valueEntry.getPath());
+            } else if (valueEntry.getPath() != null && PathProcessor.getItemAtPath(contentItem, valueEntry.getPath()).isPresent()) {
+                Optional<T> convertedPath = convertPath(contentItem, valueEntry.getPath());
+                if(convertedPath.isPresent()){
+                    return convertedPath;
+                }
             }
         }
         return Optional.empty();

src/main/java/org/bih/eos/converter/cdt/conversion_entities/Entity.java

@@ -1,9 +1,16 @@
 package org.bih.eos.converter.cdt.conversion_entities;
 
-import org.bih.eos.jpabase.entity.JPABaseEntity;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import org.bih.eos.jpabase.entity.*;
+import org.checkerframework.checker.nullness.Opt;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.core.convert.ConversionException;
 
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
 import java.util.Optional;
 import java.util.OptionalDouble;
 import java.util.function.Consumer;
@@ -23,22 +30,54 @@ public Entity(B jpaEntity) {
     public Optional<JPABaseEntity> toJpaEntity() {
         requiredOptionalFieldNull = !validateRequiredOptionalsNotNull();
         if (requiredFieldEmpty || requiredOptionalFieldNull) {
-            generateLog();
+          //  generateLog();
+            // return extractVisitOccurrence(jpaEntity);
             return Optional.empty();
         } else {
             return Optional.of(jpaEntity);
         }
     }
 
-    private void generateLog() { //TODO add property for display
-/*        ObjectMapper mapper = new ObjectMapper();
+    private Optional<JPABaseEntity> extractVisitOccurrence(B jpaEntity){
+        try {
+            Method visitOccurrence = getVisitOccurrenceMethod(jpaEntity);
+           return Optional.of((JPABaseEntity)visitOccurrence.invoke(jpaEntity));
+        } catch (NoSuchMethodException e) {
+            return Optional.empty();
+        } catch (InvocationTargetException | IllegalAccessException e) {
+            throw new RuntimeException("Reflection on getVisitOccurrence failed, check your code!");
+        }
+    }
+
+    private static Method getVisitOccurrenceMethod(Object evaluationObject) throws NoSuchMethodException {
+        if (evaluationObject.getClass() == Measurement.class) {
+            return Measurement.class.getDeclaredMethod("getVisitOccurrence");
+        } else if (evaluationObject.getClass() == ProcedureOccurrence.class) {
+            return ProcedureOccurrence.class.getDeclaredMethod("getVisitOccurrence");
+        } else if (evaluationObject.getClass() == DrugExposure.class) {
+            return DrugExposure.class.getDeclaredMethod("getVisitOccurrence");
+        } else if (evaluationObject.getClass() == Observation.class) {
+            return Observation.class.getDeclaredMethod("getVisitOccurrence");
+        } else if (evaluationObject.getClass() == ConditionOccurrence.class) {
+            return ConditionOccurrence.class.getDeclaredMethod("getVisitOccurrence");
+        } else if (evaluationObject.getClass() == DeviceExposure.class) {
+            return DeviceExposure.class.getDeclaredMethod("getVisitOccurrence");
+        }else if (evaluationObject.getClass() == Person.class) {
+            return Person.class.getDeclaredMethod("getVisitOccurrence");
+        } else {
+           throw new NoSuchMethodException();
+        }
+    }
+
+ /*   private void generateLog() { //TODO add property for display
+        ObjectMapper mapper = new ObjectMapper();
         mapper.enable(SerializationFeature.INDENT_OUTPUT);
         try {
             LOG.warn("ignore conversion since field is missing. " + mapper.writeValueAsString(jpaEntity));
         } catch (JsonProcessingException e) {
             //ignore
-        }*/
-    }
+        }
+    }*/
 
     public <T> void populateFieldIfPresent(Optional<T> optionalValue, Consumer<T> setterConsumer) {
         requiredFieldEmpty = CDMFieldSetter.setFieldIfPresent(optionalValue, requiredFieldEmpty, setterConsumer);

src/main/java/org/bih/eos/converter/cdt/converter/custom/CustomCDTConverter.java

@@ -24,7 +24,7 @@ protected List<JPABaseEntity> persist(List<JPABaseEntity> convertedEntities) {
     }
 
     protected JPABaseEntity persist(JPABaseEntity convertedEntity) {
-        return defaultConverterServices.getPersistenceService().create(convertedEntity);
+        return defaultConverterServices.getPersistenceService().createInstant(convertedEntity);
     }
 
 }

src/main/java/org/bih/eos/converter/composition/CompositionToMDConverter.java

@@ -21,6 +21,7 @@
 import java.util.*;
 
 public class CompositionToMDConverter extends CompositionToEntityConverter<CDTConverter, ConvertableComposition> {
+    HashMap<String, Integer> archetypeList = new HashMap<String, Integer>();
 
     private static final Logger LOG = LoggerFactory.getLogger(CompositionToMDConverter.class);
 
@@ -43,6 +44,7 @@ private List<JPABaseEntity> runCDTConverters(List<ConversionTrack> conversionTra
         Optional<VisitOccurrence> visitOccurrence = convertVisitOccurrence(convertableComposition.getComposition(), convertableComposition.getPerson());
         CdtExecutionParameterMedData cdtExecutionParameters = new CdtExecutionParameterMedData(convertableComposition.getPerson(), visitOccurrence);
         cdtConverterResults = iterateContent(convertableComposition.getComposition().getContent(), cdtExecutionParameters, conversionTracker, cdtConverterResults);
+        LOG.info("Mapped amount of archetypes: " + archetypeList.values().stream().mapToInt(i -> i.intValue()).sum() + ", including following archetypes: " + archetypeList);
         return cdtConverterResults;
     }
 
@@ -52,6 +54,7 @@ private List<JPABaseEntity> iterateContent(List<ContentItem> contentItems, CdtEx
             cdtExecutionParameters.setContentItem(contentItem);
             if (contentItem.getClass().equals(Section.class)) {
                 cdtConverterResults.addAll(iterateContent(((Section) contentItem).getItems(), cdtExecutionParameters, conversionTracker, cdtConverterResults));
+
                 return cdtConverterResults;
             }
             if (converterExists(cdtExecutionParameters.getArchetypeId())) {
@@ -64,6 +67,14 @@ private List<JPABaseEntity> iterateContent(List<ContentItem> contentItems, CdtEx
         return cdtConverterResults;
     }
 
+    private void increaseArchetypeCounter(String archetypeNodeId) {
+        if (archetypeList.containsKey(archetypeNodeId)) {
+            archetypeList.put(archetypeNodeId, archetypeList.get(archetypeNodeId) + 1);
+        } else {
+            archetypeList.put(archetypeNodeId, 0);
+        }
+    }
+
     private Optional<VisitOccurrence> convertVisitOccurrence(Composition composition, Person person) {
         VisitOccurrenceConverter visitOccurrenceConverter = new VisitOccurrenceConverter(defaultConverterServices);
         Optional<JPABaseEntity> visitOccurrence = visitOccurrenceConverter.convert(composition, person);
@@ -72,6 +83,7 @@ private Optional<VisitOccurrence> convertVisitOccurrence(Composition composition
 
     private List<JPABaseEntity> executeCDTConverter(CdtExecutionParameterMedData cdtExecutionParameters, List<ConversionTrack> conversionTracker) {
         List<JPABaseEntity> cdtConverterResults = new ArrayList<>();
+        increaseArchetypeCounter(cdtExecutionParameters.archetypeId);
         for (CDTConverter converter : conversionMap.get(cdtExecutionParameters.archetypeId)) {
             if (converter instanceof CDTMedicalDataConverter) {
                 List<JPABaseEntity> currentConversionResult =

src/main/java/org/bih/eos/jpabase/entity/ConditionOccurrence.java

@@ -34,19 +34,7 @@
 
 import java.util.Date;
 
-import jakarta.persistence.Access;
-import jakarta.persistence.AccessType;
-import jakarta.persistence.Column;
-import jakarta.persistence.Entity;
-import jakarta.persistence.GeneratedValue;
-import jakarta.persistence.GenerationType;
-import jakarta.persistence.Id;
-import jakarta.persistence.JoinColumn;
-import jakarta.persistence.ManyToOne;
-import jakarta.persistence.SequenceGenerator;
-import jakarta.persistence.Table;
-import jakarta.persistence.Temporal;
-import jakarta.persistence.TemporalType;
+import jakarta.persistence.*;
 
 @Entity
 @Table(name="condition_occurrence")
@@ -94,7 +82,7 @@ public class ConditionOccurrence extends JPABaseEntity {
 	@JoinColumn(name = "provider_id")
 	private Provider provider;
 
-	@ManyToOne
+	@ManyToOne(cascade = CascadeType.ALL)
 	@JoinColumn(name = "visit_occurrence_id")
 	private VisitOccurrence visitOccurrence;
 

src/main/java/org/bih/eos/jpabase/entity/DeviceExposure.java

@@ -34,19 +34,7 @@
 
 import java.util.Date;
 
-import jakarta.persistence.Access;
-import jakarta.persistence.AccessType;
-import jakarta.persistence.Column;
-import jakarta.persistence.Entity;
-import jakarta.persistence.GeneratedValue;
-import jakarta.persistence.GenerationType;
-import jakarta.persistence.Id;
-import jakarta.persistence.JoinColumn;
-import jakarta.persistence.ManyToOne;
-import jakarta.persistence.SequenceGenerator;
-import jakarta.persistence.Table;
-import jakarta.persistence.Temporal;
-import jakarta.persistence.TemporalType;
+import jakarta.persistence.*;
 
 @Entity
 @Table(name="device_exposure")
@@ -95,8 +83,8 @@ public class DeviceExposure extends JPABaseEntity {
 	@ManyToOne
 	@JoinColumn(name="provider_id")
 	private Provider provider;
-	
-	@ManyToOne
+
+	@ManyToOne(cascade = CascadeType.ALL)
 	@JoinColumn(name="visit_occurrence_id")
 	private VisitOccurrence visitOccurrence;
 

src/main/java/org/bih/eos/jpabase/entity/DrugExposure.java

@@ -34,20 +34,7 @@
 
 import java.util.Date;
 
-import jakarta.persistence.Access;
-import jakarta.persistence.AccessType;
-import jakarta.persistence.Column;
-import jakarta.persistence.Entity;
-import jakarta.persistence.FetchType;
-import jakarta.persistence.GeneratedValue;
-import jakarta.persistence.GenerationType;
-import jakarta.persistence.Id;
-import jakarta.persistence.JoinColumn;
-import jakarta.persistence.ManyToOne;
-import jakarta.persistence.SequenceGenerator;
-import jakarta.persistence.Table;
-import jakarta.persistence.Temporal;
-import jakarta.persistence.TemporalType;
+import jakarta.persistence.*;
 
 @Entity
 @Table(name="drug_exposure")
@@ -117,7 +104,7 @@ public class DrugExposure extends JPABaseEntity {
 	@JoinColumn(name = "provider_id")
 	private Provider provider;
 
-	@ManyToOne(fetch = FetchType.LAZY)
+	@ManyToOne(cascade = CascadeType.ALL)
 	@JoinColumn(name = "visit_occurrence_id")
 	private VisitOccurrence visitOccurrence;
 

src/main/java/org/bih/eos/jpabase/entity/Measurement.java

@@ -34,20 +34,7 @@
 
 import java.util.Date;
 
-import jakarta.persistence.Access;
-import jakarta.persistence.AccessType;
-import jakarta.persistence.Column;
-import jakarta.persistence.Entity;
-import jakarta.persistence.GeneratedValue;
-import jakarta.persistence.GenerationType;
-import jakarta.persistence.Id;
-import jakarta.persistence.Index;
-import jakarta.persistence.JoinColumn;
-import jakarta.persistence.ManyToOne;
-import jakarta.persistence.SequenceGenerator;
-import jakarta.persistence.Table;
-import jakarta.persistence.Temporal;
-import jakarta.persistence.TemporalType;
+import jakarta.persistence.*;
 
 @Entity
 @Table(
@@ -120,7 +107,7 @@ public class Measurement extends JPABaseEntity {
 	@JoinColumn(name="provider_id")
 	private Provider provider;
 
-	@ManyToOne
+	@ManyToOne(cascade = CascadeType.ALL)
 	@JoinColumn(name = "visit_occurrence_id")
 	private VisitOccurrence visitOccurrence;
 

src/main/java/org/bih/eos/jpabase/entity/Note.java

@@ -34,19 +34,7 @@
 
 import java.util.Date;
 
-import jakarta.persistence.Access;
-import jakarta.persistence.AccessType;
-import jakarta.persistence.Column;
-import jakarta.persistence.Entity;
-import jakarta.persistence.GeneratedValue;
-import jakarta.persistence.GenerationType;
-import jakarta.persistence.Id;
-import jakarta.persistence.JoinColumn;
-import jakarta.persistence.ManyToOne;
-import jakarta.persistence.SequenceGenerator;
-import jakarta.persistence.Table;
-import jakarta.persistence.Temporal;
-import jakarta.persistence.TemporalType;
+import jakarta.persistence.*;
 
 @Entity
 @Table(name="note")
@@ -101,7 +89,7 @@ public class Note extends JPABaseEntity {
 	@JoinColumn(name = "provider_id")
 	private Provider provider;
 
-	@ManyToOne
+	@ManyToOne(cascade = CascadeType.ALL)
 	@JoinColumn(name = "visit_occurrence_id")
 	private VisitOccurrence visitOccurrence;
 

src/main/java/org/bih/eos/jpabase/entity/Observation.java

@@ -34,20 +34,7 @@
 
 import java.util.Date;
 
-import jakarta.persistence.Access;
-import jakarta.persistence.AccessType;
-import jakarta.persistence.Column;
-import jakarta.persistence.Entity;
-import jakarta.persistence.GeneratedValue;
-import jakarta.persistence.GenerationType;
-import jakarta.persistence.Id;
-import jakarta.persistence.Index;
-import jakarta.persistence.JoinColumn;
-import jakarta.persistence.ManyToOne;
-import jakarta.persistence.SequenceGenerator;
-import jakarta.persistence.Table;
-import jakarta.persistence.Temporal;
-import jakarta.persistence.TemporalType;
+import jakarta.persistence.*;
 
 @Entity
 @Table(
@@ -107,7 +94,7 @@ public class Observation extends JPABaseEntity {
 	@JoinColumn(name = "provider_id")
 	private Provider provider;
 
-	@ManyToOne
+	@ManyToOne(cascade = CascadeType.ALL)
 	@JoinColumn(name = "visit_occurrence_id")
 	private VisitOccurrence visitOccurrence;