From 075aeb3c2812fb40b299bde1c4132bac682c10dd Mon Sep 17 00:00:00 2001 From: asp8200 Date: Wed, 14 Feb 2024 12:38:39 +0000 Subject: [PATCH 01/10] Removing all traces of docker.userEmulation --- conf/test.config | 1 - conf/test/cache.config | 1 - docs/usage.md | 4 ---- nextflow.config | 1 - 4 files changed, 7 deletions(-) diff --git a/conf/test.config b/conf/test.config index 999c3140b0..fb18881218 100644 --- a/conf/test.config +++ b/conf/test.config @@ -105,7 +105,6 @@ if (System.getenv('PROFILE')) { } else if ("$PROFILE" == "docker") { conda.enabled = false docker.enabled = true - docker.userEmulation = { params.use_gatk_spark ? false : true }.call() charliecloud.enabled = false podman.enabled = false shifter.enabled = false diff --git a/conf/test/cache.config b/conf/test/cache.config index cf63315acb..82175b2f52 100644 --- a/conf/test/cache.config +++ b/conf/test/cache.config @@ -116,7 +116,6 @@ if (System.getenv('PROFILE')) { } else if ("$PROFILE" == "docker") { conda.enabled = false docker.enabled = true - docker.userEmulation = { params.use_gatk_spark ? false : true }.call() charliecloud.enabled = false podman.enabled = false shifter.enabled = false diff --git a/docs/usage.md b/docs/usage.md index 451c69ff9d..7c4a780cc6 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -769,10 +769,6 @@ Re-start your session. Note that the way to increase the open file limit in your system may be slightly different or require additional steps. -### Cannot delete work folder when using docker + Spark - -Currently, when running spark-based tools in combination with docker, it is required to set `docker.userEmulation = false`. This can unfortunately causes permission issues when `work/` is being written with root permissions. In case this happens, you might need to configure docker to run without `userEmulation` (see [here](https://github.com/Midnighter/nf-core-adr/blob/main/docs/adr/0008-refrain-from-using-docker-useremulation-in-nextflow.md)). - ## How to handle UMIs Sarek can process UMI-reads, using [fgbio](http://fulcrumgenomics.github.io/fgbio/tools/latest/) tools. diff --git a/nextflow.config b/nextflow.config index cc1bdcbd4d..d1a0ea7ff7 100644 --- a/nextflow.config +++ b/nextflow.config @@ -187,7 +187,6 @@ profiles { charliecloud.enabled = false conda.enabled = false docker.enabled = true - docker.userEmulation = { params.use_gatk_spark ? false : true }.call() podman.enabled = false shifter.enabled = false singularity.enabled = false From 9e4ace4cbb086e563c9483404d7c2eaae0977f1e Mon Sep 17 00:00:00 2001 From: asp8200 Date: Thu, 22 Feb 2024 09:53:49 +0000 Subject: [PATCH 02/10] trigger CI-tests for gatk4 spark --- tests/test_gatk4spark.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_gatk4spark.yml b/tests/test_gatk4spark.yml index 7b6b42a3a5..b772e32b80 100644 --- a/tests/test_gatk4spark.yml +++ b/tests/test_gatk4spark.yml @@ -3,6 +3,7 @@ tags: - gatk4spark - preprocessing + - foo files: - path: results/csv/markduplicates.csv md5sum: 0dec46c4dc83acc263efc234805e9349 From 4ed689697e55faec43a356b0b997ca893d1020b3 Mon Sep 17 00:00:00 2001 From: asp8200 Date: Thu, 22 Feb 2024 10:39:23 +0000 Subject: [PATCH 03/10] trigger CI-tests for joint_germline --- tests/test_joint_germline.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_joint_germline.yml b/tests/test_joint_germline.yml index 7b634193c6..4d61124784 100644 --- a/tests/test_joint_germline.yml +++ b/tests/test_joint_germline.yml @@ -4,6 +4,7 @@ - germline - joint_germline - variant_calling + - foo files: - path: results/csv/variantcalled.csv md5sum: d2dffdbd2b4f1f26a06637592d24dab3 From 3fbbcb9271dde3f0a048fe8703eeaf91622cf796 Mon Sep 17 00:00:00 2001 From: asp8200 Date: Thu, 22 Feb 2024 21:44:58 +0000 Subject: [PATCH 04/10] Removing temporary tags --- tests/test_gatk4spark.yml | 1 - tests/test_joint_germline.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/tests/test_gatk4spark.yml b/tests/test_gatk4spark.yml index b772e32b80..7b6b42a3a5 100644 --- a/tests/test_gatk4spark.yml +++ b/tests/test_gatk4spark.yml @@ -3,7 +3,6 @@ tags: - gatk4spark - preprocessing - - foo files: - path: results/csv/markduplicates.csv md5sum: 0dec46c4dc83acc263efc234805e9349 diff --git a/tests/test_joint_germline.yml b/tests/test_joint_germline.yml index 4d61124784..7b634193c6 100644 --- a/tests/test_joint_germline.yml +++ b/tests/test_joint_germline.yml @@ -4,7 +4,6 @@ - germline - joint_germline - variant_calling - - foo files: - path: results/csv/variantcalled.csv md5sum: d2dffdbd2b4f1f26a06637592d24dab3 From 92ecd8d3f6f1422903454c6df6496a069b131c6b Mon Sep 17 00:00:00 2001 From: asp8200 Date: Thu, 22 Feb 2024 21:46:33 +0000 Subject: [PATCH 05/10] Adjusting docker.runOptions to accommodate for GATK Spark --- nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nextflow.config b/nextflow.config index c0eb9c702c..18b88037be 100644 --- a/nextflow.config +++ b/nextflow.config @@ -192,7 +192,7 @@ profiles { docker.enabled = true podman.enabled = false shifter.enabled = false - docker.runOptions = '-u $(id -u):$(id -g)' + docker.runOptions = '--platform=linux/amd64 -e "HOME=${HOME}" -v /etc/passwd:/etc/passwd:ro -v /etc/shadow:/etc/shadow:ro -v /etc/group:/etc/group:ro -v $HOME:$HOME -u $(id -u):$(id -g)' singularity.enabled = false } arm { From ca37f7d72687b1401f7e7e92601ac5c96eabdc3f Mon Sep 17 00:00:00 2001 From: asp8200 Date: Thu, 22 Feb 2024 22:17:55 +0000 Subject: [PATCH 06/10] Adding temporary tags to trigger tests --- tests/test_gatk4spark.yml | 1 + tests/test_joint_germline.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/test_gatk4spark.yml b/tests/test_gatk4spark.yml index 7b6b42a3a5..b772e32b80 100644 --- a/tests/test_gatk4spark.yml +++ b/tests/test_gatk4spark.yml @@ -3,6 +3,7 @@ tags: - gatk4spark - preprocessing + - foo files: - path: results/csv/markduplicates.csv md5sum: 0dec46c4dc83acc263efc234805e9349 diff --git a/tests/test_joint_germline.yml b/tests/test_joint_germline.yml index 7b634193c6..4d61124784 100644 --- a/tests/test_joint_germline.yml +++ b/tests/test_joint_germline.yml @@ -4,6 +4,7 @@ - germline - joint_germline - variant_calling + - foo files: - path: results/csv/variantcalled.csv md5sum: d2dffdbd2b4f1f26a06637592d24dab3 From 0688ed84c930ef4b4476a91e80c9c2d6c0258f45 Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Fri, 23 Feb 2024 13:19:34 +0100 Subject: [PATCH 07/10] Update nextflow.config --- nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nextflow.config b/nextflow.config index 18b88037be..a568830f3c 100644 --- a/nextflow.config +++ b/nextflow.config @@ -192,7 +192,7 @@ profiles { docker.enabled = true podman.enabled = false shifter.enabled = false - docker.runOptions = '--platform=linux/amd64 -e "HOME=${HOME}" -v /etc/passwd:/etc/passwd:ro -v /etc/shadow:/etc/shadow:ro -v /etc/group:/etc/group:ro -v $HOME:$HOME -u $(id -u):$(id -g)' + docker.runOptions = '-u $(id -u):$(id -g)' singularity.enabled = false } arm { From b66988886229f4513fd68bf9251f9cb28bfae17c Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Fri, 23 Feb 2024 13:19:59 +0100 Subject: [PATCH 08/10] Update nextflow.config --- nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nextflow.config b/nextflow.config index a568830f3c..c0eb9c702c 100644 --- a/nextflow.config +++ b/nextflow.config @@ -192,7 +192,7 @@ profiles { docker.enabled = true podman.enabled = false shifter.enabled = false - docker.runOptions = '-u $(id -u):$(id -g)' + docker.runOptions = '-u $(id -u):$(id -g)' singularity.enabled = false } arm { From 5238ddc0d0547568658cd1bbc8c2f98d135771f5 Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Fri, 23 Feb 2024 13:20:14 +0100 Subject: [PATCH 09/10] Apply suggestions from code review Co-authored-by: Friederike Hanssen --- tests/test_gatk4spark.yml | 1 - tests/test_joint_germline.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/tests/test_gatk4spark.yml b/tests/test_gatk4spark.yml index b772e32b80..7b6b42a3a5 100644 --- a/tests/test_gatk4spark.yml +++ b/tests/test_gatk4spark.yml @@ -3,7 +3,6 @@ tags: - gatk4spark - preprocessing - - foo files: - path: results/csv/markduplicates.csv md5sum: 0dec46c4dc83acc263efc234805e9349 diff --git a/tests/test_joint_germline.yml b/tests/test_joint_germline.yml index 4d61124784..7b634193c6 100644 --- a/tests/test_joint_germline.yml +++ b/tests/test_joint_germline.yml @@ -4,7 +4,6 @@ - germline - joint_germline - variant_calling - - foo files: - path: results/csv/variantcalled.csv md5sum: d2dffdbd2b4f1f26a06637592d24dab3 From a77e1eb7006b0bf9b2479eb549eefad5c6bf5f35 Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Fri, 23 Feb 2024 13:38:42 +0100 Subject: [PATCH 10/10] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d96249fb1a..597c462c21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Removed +- [#1405](https://github.com/nf-core/sarek/pull/1405) - Removing docker.userEmulation + ### Dependencies | Dependency | Old version | New version |