Merge branch 'dev' into release

docker/README.md

@@ -51,7 +51,7 @@ Publish the image to [Docker Hub](https://hub.docker.com/r/thehyve/transmart-api
 
 ```bash
 docker login
-TRANSMART_VERSION="17.2.3"
+TRANSMART_VERSION="17.2.4"
 docker tag transmart-api-server "thehyve/transmart-api-server:${TRANSMART_VERSION}"
 docker push "thehyve/transmart-api-server:${TRANSMART_VERSION}"
 ```

docker/docker-compose.yml

@@ -16,7 +16,7 @@ services:
 
   transmart-api-server:
     container_name: transmart-api-server
-    image: thehyve/transmart-api-server:17.2.3
+    image: thehyve/transmart-api-server:17.2.4
     ports:
       - 9081:8081
     environment:

docker/transmart-api-server/Dockerfile

@@ -1,13 +1,13 @@
 FROM openjdk:8-jre-alpine
 LABEL maintainer="[email protected]" \
       service="transmart-api-server" \
-      version="17.2.3" \
+      version="17.2.4" \
       description="This is the Docker image of the tranSMART API server"
 
 # The port is configured in the docker-entrypoint.sh when writing the config.yml
 EXPOSE 8081
 
-ENV TRANSMART_VERSION "17.2.3"
+ENV TRANSMART_VERSION "17.2.4"
 ENV TRANSMART_USER_NAME transmart
 ENV TRANSMART_GROUP_NAME "${TRANSMART_USER_NAME}"
 ENV TRANSMART_USER_HOME "/home/${TRANSMART_USER_NAME}"

docker/transmart-api-server/README.md

@@ -1,7 +1,7 @@
 # TranSMART API Server Docker Image
 
 ## Version of the API server
-The currently used version is `17.2.1`.
+The currently used version is `17.2.4`.
 
 The `Dockerfile` needs to be inspected to determine the version. It
 is apparent from the URL of the downloaded `.war` file.

docs/README.md

@@ -1,6 +1,6 @@
-# tranSMART 17.2.3 Installation
+# tranSMART 17.2.4 Installation
 
-Below are the installation instructions for tranSMART version 17.2.3. If you already have an older version of tranSMART, follow the [upgrade guide](upgrade.md) instead.
+Below are the installation instructions for tranSMART version 17.2.4. If you already have an older version of tranSMART, follow the [upgrade guide](upgrade.md) instead.
 
   1. [Prerequisites](#1-prerequisites)
   2. [Setup database](#2-setup-database)
@@ -102,7 +102,7 @@ Deployment artefacts are published to [the Nexus repository of The Hyve](https:/
 To fetch and run `transmart-api-server`:
 ```bash
 # Fetch artefacts from Maven
-TRANSMART_VERSION=17.2.3
+TRANSMART_VERSION=17.2.4
 curl -f -L https://repo.thehyve.nl/service/local/repositories/releases/content/org/transmartproject/transmart-api-server/${TRANSMART_VERSION}/transmart-api-server-${TRANSMART_VERSION}.war -o transmart-api-server-${TRANSMART_VERSION}.war && \
 # Run it with:
 java -jar -Dspring.config.location=/path/to/config.yaml transmart-api-server-${TRANSMART_VERSION}.war

transmart-copy/README.md

@@ -6,11 +6,11 @@ only substituting indexes for database identifiers for subjects, trial visits an
 
 ### Download
 The latest version can be downloaded here:
-[transmart-copy-17.2.3.jar](https://repo.thehyve.nl/service/local/repositories/releases/content/org/transmartproject/transmart-copy/17.2.3/transmart-copy-17.2.3.jar).
+[transmart-copy-17.2.4.jar](https://repo.thehyve.nl/service/local/repositories/releases/content/org/transmartproject/transmart-copy/17.2.4/transmart-copy-17.2.4.jar).
 
 ```bash
 # Download transmart-copy
-curl -f -L https://repo.thehyve.nl/service/local/repositories/releases/content/org/transmartproject/transmart-copy/17.2.3/transmart-copy-17.2.3.jar -o transmart-copy.jar
+curl -f -L https://repo.thehyve.nl/service/local/repositories/releases/content/org/transmartproject/transmart-copy/17.2.4/transmart-copy-17.2.4.jar -o transmart-copy.jar
 ```
 
 ### Usage

transmart-core-db-tests/src/integration-test/groovy/org/transmartproject/db/multidimquery/HypercubeIntegrationSpec.groovy

@@ -270,6 +270,7 @@ class HypercubeIntegrationSpec extends Specification {
         } as Set
         def expectedPatients = clinicalData.ehrClinicalFacts*.patient as Set
         def expectedVisits = clinicalData.ehrClinicalFacts*.visit as Set
+        expectedVisits.remove(null)
 
         expect:
         hypercube.dimensions.size() == clinicalData.ehrStudy.dimensions.size()

transmart-core-db-tests/src/main/groovy/org/transmartproject/db/dataquery/clinical/ClinicalTestData.groovy

@@ -146,6 +146,9 @@ class ClinicalTestData {
         def ehrStudy = StudyTestData.createStudy "ehr study", ["patient", "concept", "study", "visit"]
         def ehrClinicalFacts = createEhrFacts(conceptDims[6], visits[0..2], ehrStudy, observationStartDates, observationEndDates,
                 locations, providers)
+        // Add non-visit related observation to EHR study
+        ehrClinicalFacts << createObservationFact(conceptDims[5].conceptCode, visits[0].patient, -1, 52,
+                DUMMY_INSTANCE_ID, createTrialVisit('default', 0, null, ehrStudy))
 
         def modifierDimensions = AcrossTrialsTestData.createModifier(path: '\\Public Studies\\Medications\\Doses\\',
                 code:'TEST:DOSE', nodeType: 'F')

transmart-core-db/grails-app/domain/org/transmartproject/db/job/AsyncJobCoreDb.groovy

@@ -39,7 +39,7 @@ class AsyncJobCoreDb {
 
     static mapping = {
         id      generator: 'sequence',
-                params:     [sequence: 'hibernate_sequence', schema: 'searchapp']
+                params:     [sequence: 'async_job_seq', schema: 'i2b2demodata']
         table   'I2B2DEMODATA.ASYNC_JOB'
         version false
 

transmart-core-db/src/main/groovy/org/transmartproject/db/multidimquery/DimensionImpl.groovy

@@ -116,7 +116,7 @@ abstract class DimensionImpl<ELT,ELKey> implements Dimension {
         findDimensionDescriptionByName(name)?.dimension
     }
 
-    DimensionImpl(Size size, Density density, Packable packable, 
+    DimensionImpl(Size size, Density density, Packable packable,
                   DimensionType dimensionType = null, Integer sortIndex = null, String modifierCode = null) {
         this.size = size
         this.density = density
@@ -258,7 +258,7 @@ abstract class DimensionImpl<ELT,ELKey> implements Dimension {
 @CompileStatic @TupleConstructor
 class PropertyImpl implements Property {
     final String name; final String propertyName; final Class type
-    def get(element) { element.getAt(propertyName) }
+    def get(element) { element?.getAt(propertyName) }
 }
 
 
@@ -315,14 +315,17 @@ trait CompositeElemDim<ELT,ELKey> {
     }
 
     Map<String,Object> asSerializable(/*ELT*/ element) {
-        if(!elemType.isInstance(element)) {
+        if (element == null) {
+            return null
+        }
+        if (!elemType.isInstance(element)) {
             throw new InvalidArgumentsException("element with wrong type passed to ${this}.asSerializable; " +
                     "expected $elemType, got type ${element.class}, value $element")
         }
 
         Map result = [:]
         def pogo = (GroovyObject) element
-        for(prop in elementFields.values()) {
+        for (prop in elementFields.values()) {
             result[prop.name] = prop.get(pogo)
         }
         result

transmart-data/updatedb/release-17.2.4/README.md

@@ -0,0 +1,23 @@
+Database updates for release 17.2.4
+========================================
+
+Overview
+--------
+
+## Data migration
+
+### Update the `i2b2demodata.async_job_seq` sequence
+
+Starting from 17.2.4, the `i2b2demodata.async_job_seq` sequence is used to generate
+identifiers for the `async_job` table instead of `search_app.hibernate_sequence`.
+The `i2b2demodata.async_job_seq` sequence may need to be updated to prevent identifier
+collisions.
+
+```sql
+# Fetch highest sequence number of `i2b2demodata.async_job_seq` and `searchapp.hibernate_sequence`
+select max(count::int) from (select nextval('i2b2demodata.async_job_seq') as count union select nextval('searchapp.hibernate_sequence') as count) as counts;
+
+# Update the start value of the `async_job_seq` sequence. Use the result of the previous query
+# instead of `<VALUE>`
+alter sequence i2b2demodata.async_job_seq start with <VALUE>;
+```

transmart-rest-api/src/integration-test/groovy/org/transmartproject/rest/DataTableViewDataSerializationServiceSpec.groovy

@@ -19,7 +19,6 @@ import org.transmartproject.db.i2b2data.ConceptDimension
 import org.transmartproject.db.multidimquery.DimensionImpl
 import org.transmartproject.db.multidimquery.PropertyImpl
 import org.transmartproject.mock.MockUser
-import org.transmartproject.core.multidimquery.export.Format
 import org.transmartproject.rest.serialization.tabular.DataTableTSVSerializer
 import spock.lang.Specification
 
@@ -74,6 +73,34 @@ class DataTableViewDataSerializationServiceSpec extends Specification {
                 files.concept)
     }
 
+    void testVisitSerialization() {
+        setupData()
+
+        def file = new ByteArrayOutputStream()
+        def zipFile = new ZipOutputStream(file)
+        Constraint constraint = new StudyNameConstraint(studyId: clinicalData.ehrStudy.studyId)
+        def tableConfig = new TableConfig(
+                rowDimensions: ['patient', 'visit'],
+                columnDimensions: ['concept']
+        )
+
+        serializationService.writeTableToTsv(constraint, tableConfig, adminUser, zipFile)
+        def files = parseTSVZip(file)
+
+        expect:
+        'data' in files
+        // Check data file
+        def visitIds = clinicalData.ehrClinicalFacts*.visit.collect { it?.id?.toString() }.unique()
+        files.data == [[/* Patient Id */ '', /* Visit Id */ '', /* Value for non-visit observations */ 'c6', /* Multiple observations for concept */ 'c7'],
+                       ['-103', '', '52.0000000000000000', ''],
+                       ['-103', visitIds[0], '', '-45.4200000000000000;-45.4200000000000000;-45.4200000000000000;-45.4200000000000000'],
+                       ['-103', visitIds[1], '', '-45.4200000000000000;-45.4200000000000000;-45.4200000000000000;-45.4200000000000000'],
+                       ['-103', visitIds[2], '', '-45.4200000000000000;-45.4200000000000000;-45.4200000000000000;-45.4200000000000000']]
+        // Check visit ids in visit file
+        files.visit[0][1] == 'id'
+        visitIds == files.visit[1..-1].collect { it ? it[1] : it } + [null]
+    }
+
     void testTSVDimensionSerialization() {
         // This is actually more of a unittest, so no setupData()
 
@@ -118,6 +145,16 @@ class DataTableViewDataSerializationServiceSpec extends Specification {
                 ['2', 'no2', '42', ''],
                 ['3', 'no3', '', '44']
         ]
+        // Test nullable dimensions
+        roundTrip(dim, [
+                [id: 1, name: 'no1'],
+                null,
+                [id: 2, name: 'no2']
+        ]) == [
+                ['id', 'name'],
+                ['1', 'no1'],
+                ['2', 'no2'],
+        ]
     }
 
     List<List<String>> roundTrip(Dimension dim, List elements) {
@@ -180,7 +217,7 @@ class DataTableViewDataSerializationServiceSpec extends Specification {
         }
 
         assert elementsMap as Set ==
-                elements.collect { dim.asSerializable(it).collectEntries(valuesToString) } as Set
+                elements.collect { dim.asSerializable(it)?.collectEntries(valuesToString) } as Set
         return true
     }
 

transmart-schemas/src/main/resources/db/changelog/changes/20190926092415_async_job_seq.xml

@@ -0,0 +1,13 @@
+<?xml version="1.1" encoding="UTF-8" standalone="no"?>
+<!--
+  ~ Copyright (c) 2019  The Hyve B.V.
+  ~  This file is distributed under the GNU General Public License
+  ~  (see accompanying file LICENSE).
+  -->
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.6.xsd">
+    <changeSet author="[email protected]" id="20190926092415-0">
+        <createSequence sequenceName="async_job_seq" schemaName="i2b2demodata"/>
+    </changeSet>
+</databaseChangeLog>

transmart-schemas/src/main/resources/db/changelog/db.changelog-master.yaml

@@ -11,4 +11,6 @@ databaseChangeLog:
       file: classpath:db/changelog/changes/20190625151656_create_pg_bitcount_extension.xml
   - include:
       file: classpath:db/changelog/changes/20191907041159_biomart_user_views.xml
+  - include:
+      file: classpath:db/changelog/changes/20190926092415_async_job_seq.xml
 relativeToChangelogFile: false