Merge branch 'release-2.4.x' of github.com:opencb/opencga into releas…

…e-2.4.x

opencga-catalog/src/main/java/org/opencb/opencga/catalog/db/mongodb/iterators/ClinicalAnalysisCatalogMongoDBIterator.java

@@ -28,9 +28,11 @@
 import org.opencb.opencga.catalog.exceptions.CatalogAuthorizationException;
 import org.opencb.opencga.catalog.exceptions.CatalogDBException;
 import org.opencb.opencga.catalog.exceptions.CatalogParameterException;
+import org.opencb.opencga.core.common.JacksonUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.IOException;
 import java.util.*;
 
 import static org.opencb.opencga.catalog.db.api.ClinicalAnalysisDBAdaptor.QueryParams.*;
@@ -246,6 +248,13 @@ private Document fillIndividualData(Document individualDoc, Map<String, Document
     }
 
     private Document fillIndividualData(Document individualDoc, Document completeIndividualDoc) {
+        Document individualCopy;
+        try {
+            individualCopy = JacksonUtils.copy(completeIndividualDoc, Document.class);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+
         List<Document> samples = (List<Document>) individualDoc.get(IndividualDBAdaptor.QueryParams.SAMPLES.key());
         List<Document> completeSamples = (List<Document>) completeIndividualDoc.get(IndividualDBAdaptor.QueryParams.SAMPLES.key());
 
@@ -263,10 +272,10 @@ private Document fillIndividualData(Document individualDoc, Document completeInd
                 }
             }
 
-            completeIndividualDoc.put(IndividualDBAdaptor.QueryParams.SAMPLES.key(), finalSamples);
+            individualCopy.put(IndividualDBAdaptor.QueryParams.SAMPLES.key(), finalSamples);
         }
 
-        return completeIndividualDoc;
+        return individualCopy;
     }
 
     private void fillInterpretationData(Document clinicalAnalysis, Map<String, Document> interpretationMap) {

opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/FamilyManager.java

@@ -1438,31 +1438,21 @@ private void validateFamily(Family family, List<Individual> existingMembers) thr
                 membersMap.put(String.valueOf(individual.getUid()), individual);
             }
 
-            String parentsKey = null;
+            String parentsKey = "";
             if (individual.getMother() != null) {
-                if (individual.getMother().getUid() > 0) {
-                    individual.getMother().setId(String.valueOf(individual.getMother().getUid()));
-                }
                 if (!StringUtils.isEmpty(individual.getMother().getId())) {
-                    parentsKey = individual.getMother().getId() + "||F";
+                    parentsKey += individual.getMother().getId() + "__" + individual.getMother().getUid() + "||F";
                 }
             }
             if (individual.getFather() != null) {
-                if (parentsKey != null) {
+                if (StringUtils.isNotEmpty(parentsKey)) {
                     parentsKey += "---";
                 }
-                if (individual.getFather().getUid() > 0) {
-                    individual.getFather().setId(String.valueOf(individual.getFather().getUid()));
-                }
                 if (!StringUtils.isEmpty(individual.getFather().getId())) {
-                    if (parentsKey != null) {
-                        parentsKey += individual.getFather().getId() + "||M";
-                    } else {
-                        parentsKey = individual.getFather().getId() + "||M";
-                    }
+                    parentsKey += individual.getFather().getId() + "__" + individual.getFather().getUid() + "||M";
                 }
             }
-            if (parentsKey == null) {
+            if (StringUtils.isEmpty(parentsKey)) {
                 noParentsSet.add(individual.getId());
             } else {
                 if (!parentsMap.containsKey(parentsKey)) {
@@ -1474,32 +1464,31 @@ private void validateFamily(Family family, List<Individual> existingMembers) thr
 
         // 2. Loop over the parentsMap object. We will be emptying the noParentsSet as soon as we find a parent in the set. Once,
         // everything finishes, that set should be empty. Otherwise, it will mean that parent is not in use
-        // On the other hand, all the parents should exist in the membersMap, otherwise it will mean that is missing in the family
         for (Map.Entry<String, List<Individual>> parentListEntry : parentsMap.entrySet()) {
             String[] split = parentListEntry.getKey().split("---");
             for (String parentName : split) {
                 String[] splitNameSex = parentName.split("\\|\\|");
-                String name = splitNameSex[0];
+                String[] splitIdUid = splitNameSex[0].split("__");
+                String parentId = splitIdUid[0];
+                String parentUid = splitIdUid[1];
                 SexOntologyTermAnnotation sexTerm = splitNameSex[1].equals("F")
                         ? SexOntologyTermAnnotation.initFemale()
                         : SexOntologyTermAnnotation.initMale();
                 IndividualProperty.Sex sex = sexTerm.getSex();
 
-                if (!membersMap.containsKey(name)) {
-                    throw new CatalogException("The parent " + name + " is not present in the members list");
-                } else {
+                if (membersMap.containsKey(parentUid)) {
                     // Check if the sex is correct
-                    IndividualProperty.Sex sex1 = membersMap.get(name).getSex() != null
-                            ? membersMap.get(name).getSex().getSex()
+                    IndividualProperty.Sex sex1 = membersMap.get(parentUid).getSex() != null
+                            ? membersMap.get(parentUid).getSex().getSex()
                             : IndividualProperty.Sex.UNKNOWN;
                     if (sex1 != null && sex1 != sex && sex1 != IndividualProperty.Sex.UNKNOWN) {
-                        throw new CatalogException("Sex of parent " + name + " is incorrect or the relationship is incorrect. In "
+                        throw new CatalogException("Sex of parent '" + parentId + "' is incorrect or the relationship is incorrect. In "
                                 + "principle, it should be " + sexTerm);
                     }
-                    membersMap.get(name).setSex(sexTerm);
+                    membersMap.get(parentUid).setSex(sexTerm);
 
                     // We attempt to remove the individual from the noParentsSet
-                    noParentsSet.remove(membersMap.get(name).getId());
+                    noParentsSet.remove(membersMap.get(parentUid).getId());
                 }
             }
         }

opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/ClinicalAnalysisManagerTest.java

@@ -45,6 +45,7 @@
 import org.opencb.opencga.catalog.utils.Constants;
 import org.opencb.opencga.catalog.utils.ParamUtils;
 import org.opencb.opencga.core.api.ParamConstants;
+import org.opencb.opencga.core.common.JacksonUtils;
 import org.opencb.opencga.core.common.TimeUtils;
 import org.opencb.opencga.core.models.AclEntryList;
 import org.opencb.opencga.core.models.AclParams;
@@ -3206,4 +3207,34 @@ public void updateClinicalAnalysisFiles() throws CatalogException {
         assertTrue(files.subList(2, 4).stream().map(File::getPath).collect(Collectors.toSet())
                 .containsAll(ca.getFiles().stream().map(File::getPath).collect(Collectors.toSet())));
     }
+
+    @Test
+    public void fetchCasesWithSameProbandAndDifferentSample() throws CatalogException, IOException {
+        Sample sample1 = DummyModelUtils.getDummySample("sample1");
+        Sample sample2 = DummyModelUtils.getDummySample("sample2");
+
+        Individual proband = DummyModelUtils.getDummyIndividual("proband", Arrays.asList(sample1, sample2), null, null);
+        catalogManager.getIndividualManager().create(STUDY, proband, QueryOptions.empty(), sessionIdUser);
+
+        Individual probandCopy = JacksonUtils.copy(proband, Individual.class);
+        probandCopy.setSamples(Collections.singletonList(proband.getSamples().get(0)));
+        ClinicalAnalysis case1 = DummyModelUtils.getDummyClinicalAnalysis("case1", probandCopy, null, null);
+        catalogManager.getClinicalAnalysisManager().create(STUDY, case1, QueryOptions.empty(), sessionIdUser);
+
+        probandCopy.setSamples(Collections.singletonList(proband.getSamples().get(1)));
+        ClinicalAnalysis case2 = DummyModelUtils.getDummyClinicalAnalysis("case2", probandCopy, null, null);
+        catalogManager.getClinicalAnalysisManager().create(STUDY, case2, QueryOptions.empty(), sessionIdUser);
+
+        OpenCGAResult<ClinicalAnalysis> result = catalogManager.getClinicalAnalysisManager().search(STUDY, new Query(),
+                QueryOptions.empty(), sessionIdUser);
+        assertEquals(2, result.getNumResults());
+        assertEquals(case1.getId(), result.getResults().get(0).getId());
+        assertEquals(proband.getId(), result.getResults().get(0).getProband().getId());
+        assertEquals(1, result.getResults().get(0).getProband().getSamples().size());
+        assertEquals(proband.getSamples().get(0).getId(), result.getResults().get(0).getProband().getSamples().get(0).getId());
+        assertEquals(case2.getId(), result.getResults().get(1).getId());
+        assertEquals(proband.getId(), result.getResults().get(1).getProband().getId());
+        assertEquals(1, result.getResults().get(1).getProband().getSamples().size());
+        assertEquals(proband.getSamples().get(1).getId(), result.getResults().get(1).getProband().getSamples().get(0).getId());
+    }
 }

opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/DummyModelUtils.java

@@ -179,12 +179,17 @@ public static ClinicalAnalysis getDummyClinicalAnalysis(Individual proband, Fami
     }
 
     public static ClinicalAnalysis getDummyClinicalAnalysis(String id, Individual proband, Family family, List<Panel> panelList) {
-        return new ClinicalAnalysis()
+        ClinicalAnalysis clinicalAnalysis = new ClinicalAnalysis()
                 .setId(id)
-                .setType(ClinicalAnalysis.Type.FAMILY)
                 .setProband(proband)
                 .setFamily(family)
                 .setPanels(panelList);
+        if (family != null) {
+            clinicalAnalysis.setType(ClinicalAnalysis.Type.FAMILY);
+        } else {
+            clinicalAnalysis.setType(ClinicalAnalysis.Type.SINGLE);
+        }
+        return clinicalAnalysis;
     }
 
     public static Interpretation getDummyInterpretation(List<Panel> panelList) {

opencga-catalog/src/test/java/org/opencb/opencga/catalog/managers/FamilyManagerTest.java

@@ -1373,4 +1373,27 @@ public void updateDeleteInUseInCATest() throws CatalogException {
         }
     }
 
+    @Test
+    public void updateFamilyMembers() throws CatalogException {
+        Individual child = DummyModelUtils.getDummyIndividual("child", null, null, null);
+        Individual father = DummyModelUtils.getDummyIndividual("father", null, null, null);
+        Individual mother = DummyModelUtils.getDummyIndividual("mother", null, null, null);
+        child.setFather(father);
+        child.setMother(mother);
+
+        catalogManager.getIndividualManager().create(STUDY, father, QueryOptions.empty(), sessionIdUser);
+        catalogManager.getIndividualManager().create(STUDY, mother, QueryOptions.empty(), sessionIdUser);
+        catalogManager.getIndividualManager().create(STUDY, child, QueryOptions.empty(), sessionIdUser);
+
+        Family family = DummyModelUtils.getDummyFamily("family");
+        family.setMembers(null);
+        catalogManager.getFamilyManager().create(STUDY, family, Collections.singletonList(child.getId()), QueryOptions.empty(), sessionIdUser);
+
+        FamilyUpdateParams updateParams = new FamilyUpdateParams().setMembers(Arrays.asList(
+                new IndividualReferenceParam(child.getId(), child.getUuid()),
+                new IndividualReferenceParam(father.getId(), child.getUuid())
+        ));
+        catalogManager.getFamilyManager().update(STUDY, family.getId(), updateParams, QueryOptions.empty(), sessionIdUser);
+    }
+
 }

opencga-master/src/main/java/org/opencb/opencga/master/monitor/executors/K8SExecutor.java

@@ -49,6 +49,7 @@ public class K8SExecutor implements BatchExecutor {
     public static final String K8S_IMAGE_PULL_POLICY = "k8s.imagePullPolicy";
     public static final String K8S_IMAGE_PULL_SECRETS = "k8s.imagePullSecrets";
     public static final String K8S_TTL_SECONDS_AFTER_FINISHED = "k8s.ttlSecondsAfterFinished";
+    public static final String K8S_DIND_ROOTLESS = "k8s.dind.rootless";
     public static final String K8S_DIND_IMAGE_NAME = "k8s.dind.imageName";
     public static final String K8S_REQUESTS = "k8s.requests";
     public static final String K8S_LIMITS = "k8s.limits";
@@ -83,7 +84,6 @@ public class K8SExecutor implements BatchExecutor {
             .build();
     private static final String DIND_DONE_FILE = "/usr/share/pod/done";
 
-    private final String k8sClusterMaster;
     private final String namespace;
     private final String imageName;
     private final List<VolumeMount> volumeMounts;
@@ -101,13 +101,13 @@ public class K8SExecutor implements BatchExecutor {
     private final Map<String, Pair<Instant, String>> jobStatusCache = new ConcurrentHashMap<>();
     private final Watch podsWatcher;
     private final Watch jobsWatcher;
-    private String imagePullPolicy;
-    private List<LocalObjectReference> imagePullSecrets;
-    private int ttlSecondsAfterFinished;
+    private final String imagePullPolicy;
+    private final List<LocalObjectReference> imagePullSecrets;
+    private final int ttlSecondsAfterFinished;
 
     public K8SExecutor(Configuration configuration) {
         Execution execution = configuration.getAnalysis().getExecution();
-        this.k8sClusterMaster = execution.getOptions().getString(K8S_MASTER_NODE);
+        String k8sClusterMaster = execution.getOptions().getString(K8S_MASTER_NODE);
         this.namespace = execution.getOptions().getString(K8S_NAMESPACE);
         this.imageName = execution.getOptions().getString(K8S_IMAGE_NAME);
         this.volumeMounts = buildVolumeMounts(execution.getOptions().getList(K8S_VOLUME_MOUNTS));
@@ -171,13 +171,25 @@ public K8SExecutor(Configuration configuration) {
         }
 
         String dindImageName = execution.getOptions().getString(K8S_DIND_IMAGE_NAME, "docker:dind-rootless");
+        boolean rootless;
+        if (execution.getOptions().containsKey(K8S_DIND_ROOTLESS)) {
+            rootless = execution.getOptions().getBoolean(K8S_DIND_ROOTLESS);
+        } else {
+            rootless = dindImageName.contains("dind-rootless");
+        }
+        SecurityContext dindSecurityContext;
+        if (rootless) {
+            dindSecurityContext = new SecurityContextBuilder()
+                    .withRunAsNonRoot(true)
+                    .withRunAsUser(1000L)
+                    .withPrivileged(true).build();
+        } else {
+            dindSecurityContext = new SecurityContextBuilder().withPrivileged(true).build();
+        }
         dockerDaemonSidecar = new ContainerBuilder()
                 .withName("dind-daemon")
                 .withImage(dindImageName)
-                .withSecurityContext(new SecurityContextBuilder()
-                        .withRunAsNonRoot(true)
-                        .withRunAsUser(1000L)
-                        .withPrivileged(true).build())
+                .withSecurityContext(dindSecurityContext)
                 .withEnv(new EnvVar("DOCKER_TLS_CERTDIR", "", null))
 //                .withResources(resources) // TODO: Should we add resources here?
                 .withCommand("/bin/sh", "-c")

opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/main/java/org/opencb/opencga/storage/hadoop/variant/io/HadoopVariantExporter.java

@@ -75,7 +75,9 @@ public URI export(@Nullable URI outputFileUri, VariantWriterFactory.VariantOutpu
         Query query = variantQuery.getQuery();
         QueryOptions queryOptions = variantQuery.getInputOptions();
         boolean smallQuery = false;
-        if (!queryOptions.getBoolean("skipSmallQuery", false)) {
+        boolean parquet = outputFormat == VariantWriterFactory.VariantOutputFormat.PARQUET_GZ
+                || outputFormat == VariantWriterFactory.VariantOutputFormat.PARQUET;
+        if (!queryOptions.getBoolean("skipSmallQuery", false) && !parquet) {
             ParsedVariantQuery.VariantQueryXref xrefs = VariantQueryParser.parseXrefs(query);
             if (xrefs.getVariants().size() > 0 && xrefs.getVariants().size() < 2000) {
                 // FIXME: Is this scenario still needed?

opencga-storage/opencga-storage-hadoop/opencga-storage-hadoop-core/src/test/java/org/opencb/opencga/storage/hadoop/variant/io/HadoopVariantExporterTest.java

@@ -15,6 +15,7 @@
 import org.opencb.opencga.storage.core.metadata.models.StudyMetadata;
 import org.opencb.opencga.storage.core.variant.VariantStorageBaseTest;
 import org.opencb.opencga.storage.core.variant.VariantStorageOptions;
+import org.opencb.opencga.storage.core.variant.adaptors.VariantQuery;
 import org.opencb.opencga.storage.core.variant.io.VariantExporter;
 import org.opencb.opencga.storage.core.variant.io.VariantWriterFactory;
 import org.opencb.opencga.storage.core.variant.solr.VariantSolrExternalResource;
@@ -125,9 +126,9 @@ public void exportMultiRegion() throws Exception {
     public void exportAvroGz() throws Exception {
         String fileName = "variants.avro_gz";
         URI uri = getOutputUri(fileName);
-        variantStorageEngine.exportData(uri, VariantWriterFactory.VariantOutputFormat.AVRO_GZ, null, new Query(STUDY.key(), study1), new QueryOptions());
+        uri = variantStorageEngine.exportData(uri, VariantWriterFactory.VariantOutputFormat.AVRO_GZ, null, new Query(STUDY.key(), study1), new QueryOptions());
 
-        copyToLocal(fileName, uri);
+        copyToLocal(uri);
     }
 
     @Test
@@ -154,7 +155,7 @@ public void exportTped() throws Exception {
         URI uri = getOutputUri(fileName);
         uri = variantStorageEngine.exportData(uri, VariantWriterFactory.VariantOutputFormat.TPED, null, new Query(STUDY.key(), study1), new QueryOptions());
 
-        copyToLocal(Paths.get(uri).getFileName().toString(), uri);
+        copyToLocal(uri);
     }
 
     @Test
@@ -170,9 +171,19 @@ public void exportJson() throws Exception {
     public void exportParquet() throws Exception {
         String fileName = "variants.parquet";
         URI uri = getOutputUri(fileName);
-        variantStorageEngine.exportData(uri, VariantWriterFactory.VariantOutputFormat.PARQUET_GZ, null, new Query(STUDY.key(), study1), new QueryOptions());
+        uri = variantStorageEngine.exportData(uri, VariantWriterFactory.VariantOutputFormat.PARQUET_GZ, null, new Query(STUDY.key(), study1), new QueryOptions());
 
-        copyToLocal(fileName, uri);
+        copyToLocal(uri);
+    }
+
+    @Test
+    public void exportParquetSmallQuery() throws Exception {
+        String fileName = "variants.small.parquet";
+        URI uri = getOutputUri(fileName);
+        uri = variantStorageEngine.exportData(uri, VariantWriterFactory.VariantOutputFormat.PARQUET_GZ, null, new VariantQuery()
+                .study(study1).sample("NA12877"), new QueryOptions());
+
+        copyToLocal(uri);
     }
 
     @Test
@@ -256,8 +267,13 @@ public void exportWithGenes() throws Exception {
         copyToLocal(fileName, uri);
     }
 
+    protected void copyToLocal(URI uri) throws IOException {
+        copyToLocal(Paths.get(uri.getPath()).getFileName().toString(), uri);
+    }
+
     protected void copyToLocal(String fileName, URI uri) throws IOException {
         if (!exportToLocal) {
+            System.out.println("Copy file " + uri);
             FileSystem.get(externalResource.getConf()).copyToLocalFile(true,
                     new Path(uri),
                     new Path(outputUri.resolve(fileName)));