feat: add gnomAD SV/CNV v4 support (#89)

* feat: add gnomAD SV/CNV v4 support * make linter happy
varfish-org · Jun 19, 2024 · c5825f0 · c5825f0
1 parent 83dd2a7
commit c5825f0
Show file tree

Hide file tree

Showing 66 changed files with 370 additions and 211 deletions.
diff --git a/Snakefile b/Snakefile
@@ -127,7 +127,6 @@ rule all:
         f"work/annos/grch38/seqvars/gnomad_mtdna/{DV.gnomad_mtdna}/gnomad_mtdna.vcf.gz",
         f"work/download/annos/grch38/seqvars/gnomad_exomes/{DV.gnomad_v4}/.done",
         f"work/download/annos/grch38/seqvars/gnomad_genomes/{DV.gnomad_v4}/.done",
-        # NB: gnomAD-SV GRCh38 was announced end of 2020 but not released yet
         # -- genome browser "features" (position-specific)
         # ---- GRCh37
         f"work/annos/grch37/features/cons/{DV.ucsc_cons_37}/ucsc_conservation.tsv",
@@ -181,10 +180,6 @@ rule all:
         # ----- genes
         f"output/full/annonars/genes-{DV.acmg_sf}+{DV.gnomad_constraints}+{DV.dbnsfp}+{DV.hpo}+{DV.today}+{PV.annonars}/rocksdb/IDENTITY",
         # -- worker data
-        f"output/full/worker/genes-regions-grch37-{DV.refseq_37}+{PV.worker}/refseq_genes.bin",
-        f"output/full/worker/genes-regions-grch37-{DV.ensembl_37}+{PV.worker}/ensembl_genes.bin",
-        f"output/full/worker/genes-regions-grch38-{DV.refseq_38}+{PV.worker}/refseq_genes.bin",
-        f"output/full/worker/genes-regions-grch38-{DV.ensembl_38}+{PV.worker}/ensembl_genes.bin",
         f"output/full/worker/genes-xlink-{DV.today}+{PV.worker}/genes-xlink.bin",
         f"output/full/worker/acmg-sf-{DV.acmg_sf}+{PV.worker}/acmg_sf.tsv",
         f"output/full/worker/mim2gene-{DV.today}+{PV.worker}/mim2gene.tsv",
@@ -200,6 +195,8 @@ rule all:
         f"output/full/worker/bgdb-dgv-gs-grch38-{DV.dgv}+{PV.worker}/bgdb-dgv-gs.bin",
         f"output/full/worker/bgdb-gnomad-grch37-{DV.gnomad_sv}+{PV.worker}/bgdb-gnomad.bin",
         f"output/full/worker/bgdb-exac-grch37-{DV.exac_cnv}+{PV.worker}/bgdb-exac.bin",
+        f"output/full/worker/bgdb-gnomad-exomes-cnv-grch38-{DV.gnomad_sv4}+{PV.worker}/bgdb-gnomad-exomes-cnv-grch38.bin",
+        f"output/full/worker/bgdb-gnomad-genomes-sv-grch38-{DV.gnomad_sv4}+{PV.worker}/bgdb-gnomad-genomes-sv-grch38.bin",
         f"output/full/worker/bgdb-g1k-grch37-{DV.g1k_svs}+{PV.worker}/bgdb-g1k.bin",
         f"output/full/worker/clinvar-strucvars-grch37-{DV.clinvar_version}+{PV.worker}/clinvar-strucvars.bin",
         f"output/full/worker/clinvar-strucvars-grch38-{DV.clinvar_version}+{PV.worker}/clinvar-strucvars.bin",
@@ -227,6 +224,8 @@ rule all:
         f"output/full/tracks/track-strucvars-exac-grch37-{DV.exac_cnv}+{DV.tracks}/exac.bed.gz",
         f"output/full/tracks/track-strucvars-g1k-grch37-{DV.g1k_svs}+{DV.tracks}/g1k.bed.gz",
         f"output/full/tracks/track-strucvars-gnomad-grch37-{DV.gnomad_sv}+{DV.tracks}/gnomad.bed.gz",
+        f"output/full/tracks/track-strucvars-gnomad-sv-grch38-{DV.gnomad_sv4}+{DV.tracks}/gnomad-sv.bed.gz",
+        f"output/full/tracks/track-strucvars-gnomad-cnv-grch38-{DV.gnomad_cnv4}+{DV.tracks}/gnomad-cnv.bed.gz",
         # ----- known pathogenic MMS
         f"output/full/tracks/track-strucvars-patho-mms-grch37-{DV.patho_mms}+{DV.tracks}/patho-mms.bed",
         f"output/full/tracks/track-strucvars-patho-mms-grch38-{DV.patho_mms}+{DV.tracks}/patho-mms.bed",
@@ -408,7 +407,6 @@ include: "rules/output/annonars/regions.smk"
 # ---- worker
 include: "rules/output/worker/patho_mms.smk"
 include: "rules/output/worker/clinvar.smk"
-include: "rules/output/worker/genes_regions.smk"
 include: "rules/output/worker/hgnc.smk"
 include: "rules/output/worker/acmg.smk"
 include: "rules/output/worker/mim2gene.smk"

diff --git a/download_urls.yml b/download_urls.yml
@@ -105,29 +105,101 @@
     strategy: gz-head
     count: 160
 - url: https://storage.googleapis.com/gcp-public-data--gnomad/release/4.0/genome_sv/gnomad.v4.0.sv.chr1.vcf.gz
+  excerpt_strategy:
+    strategy: vcf-head
+    count: 1800
 - url: https://storage.googleapis.com/gcp-public-data--gnomad/release/4.0/genome_sv/gnomad.v4.0.sv.chr2.vcf.gz
+  excerpt_strategy:
+    strategy: vcf-head
+    count: 1800
 - url: https://storage.googleapis.com/gcp-public-data--gnomad/release/4.0/genome_sv/gnomad.v4.0.sv.chr3.vcf.gz
+  excerpt_strategy:
+    strategy: vcf-head
+    count: 1800
 - url: https://storage.googleapis.com/gcp-public-data--gnomad/release/4.0/genome_sv/gnomad.v4.0.sv.chr4.vcf.gz
+  excerpt_strategy:
+    strategy: vcf-head
+    count: 1800
 - url: https://storage.googleapis.com/gcp-public-data--gnomad/release/4.0/genome_sv/gnomad.v4.0.sv.chr5.vcf.gz
+  excerpt_strategy:
+    strategy: vcf-head
+    count: 1800
 - url: https://storage.googleapis.com/gcp-public-data--gnomad/release/4.0/genome_sv/gnomad.v4.0.sv.chr6.vcf.gz
+  excerpt_strategy:
+    strategy: vcf-head
+    count: 1800
 - url: https://storage.googleapis.com/gcp-public-data--gnomad/release/4.0/genome_sv/gnomad.v4.0.sv.chr7.vcf.gz
+  excerpt_strategy:
+    strategy: vcf-head
+    count: 1800
 - url: https://storage.googleapis.com/gcp-public-data--gnomad/release/4.0/genome_sv/gnomad.v4.0.sv.chr8.vcf.gz
+  excerpt_strategy:
+    strategy: vcf-head
+    count: 1800
 - url: https://storage.googleapis.com/gcp-public-data--gnomad/release/4.0/genome_sv/gnomad.v4.0.sv.chr9.vcf.gz
+  excerpt_strategy:
+    strategy: vcf-head
+    count: 1800
 - url: https://storage.googleapis.com/gcp-public-data--gnomad/release/4.0/genome_sv/gnomad.v4.0.sv.chr10.vcf.gz
+  excerpt_strategy:
+    strategy: vcf-head
+    count: 1800
 - url: https://storage.googleapis.com/gcp-public-data--gnomad/release/4.0/genome_sv/gnomad.v4.0.sv.chr11.vcf.gz
+  excerpt_strategy:
+    strategy: vcf-head
+    count: 1800
 - url: https://storage.googleapis.com/gcp-public-data--gnomad/release/4.0/genome_sv/gnomad.v4.0.sv.chr12.vcf.gz
+  excerpt_strategy:
+    strategy: vcf-head
+    count: 1800
 - url: https://storage.googleapis.com/gcp-public-data--gnomad/release/4.0/genome_sv/gnomad.v4.0.sv.chr13.vcf.gz
+  excerpt_strategy:
+    strategy: vcf-head
+    count: 1800
 - url: https://storage.googleapis.com/gcp-public-data--gnomad/release/4.0/genome_sv/gnomad.v4.0.sv.chr14.vcf.gz
+  excerpt_strategy:
+    strategy: vcf-head
+    count: 1800
 - url: https://storage.googleapis.com/gcp-public-data--gnomad/release/4.0/genome_sv/gnomad.v4.0.sv.chr15.vcf.gz
+  excerpt_strategy:
+    strategy: vcf-head
+    count: 1800
 - url: https://storage.googleapis.com/gcp-public-data--gnomad/release/4.0/genome_sv/gnomad.v4.0.sv.chr16.vcf.gz
+  excerpt_strategy:
+    strategy: vcf-head
+    count: 1800
 - url: https://storage.googleapis.com/gcp-public-data--gnomad/release/4.0/genome_sv/gnomad.v4.0.sv.chr17.vcf.gz
+  excerpt_strategy:
+    strategy: vcf-head
+    count: 1800
 - url: https://storage.googleapis.com/gcp-public-data--gnomad/release/4.0/genome_sv/gnomad.v4.0.sv.chr18.vcf.gz
+  excerpt_strategy:
+    strategy: vcf-head
+    count: 1800
 - url: https://storage.googleapis.com/gcp-public-data--gnomad/release/4.0/genome_sv/gnomad.v4.0.sv.chr19.vcf.gz
+  excerpt_strategy:
+    strategy: vcf-head
+    count: 1800
 - url: https://storage.googleapis.com/gcp-public-data--gnomad/release/4.0/genome_sv/gnomad.v4.0.sv.chr20.vcf.gz
+  excerpt_strategy:
+    strategy: vcf-head
+    count: 1800
 - url: https://storage.googleapis.com/gcp-public-data--gnomad/release/4.0/genome_sv/gnomad.v4.0.sv.chr21.vcf.gz
+  excerpt_strategy:
+    strategy: vcf-head
+    count: 1800
 - url: https://storage.googleapis.com/gcp-public-data--gnomad/release/4.0/genome_sv/gnomad.v4.0.sv.chr22.vcf.gz
+  excerpt_strategy:
+    strategy: vcf-head
+    count: 1800
 - url: https://storage.googleapis.com/gcp-public-data--gnomad/release/4.0/genome_sv/gnomad.v4.0.sv.chrX.vcf.gz
+  excerpt_strategy:
+    strategy: vcf-head
+    count: 1800
 - url: https://storage.googleapis.com/gcp-public-data--gnomad/release/4.0/genome_sv/gnomad.v4.0.sv.chrY.vcf.gz
+  excerpt_strategy:
+    strategy: vcf-head
+    count: 1800
 
 - url: https://www.deciphergenomics.org/files/downloads/HI_Predictions_Version3.bed.gz
 
@@ -157,13 +229,9 @@
   url: https://search.clinicalgenome.org/kb/reports/curation-activity-summary-report
   skip_upstream_check: true  # does not work reliably in tests
 
-- url: https://github.com/bihealth/annonars-data-clinvar/releases/download/clinvar-weekly-20230625/clinvar-strucvar-grch37-2023-0625+0.6.3.tar.gz
+- url: https://github.com/varfish-org/clinvar-data-jsonl/releases/download/clinvar-weekly-20240612/clinvar-data-extract-vars-20240612+0.17.0.tar.gz
   excerpt_strategy:
-    strategy: no-excerpt
-    count: null
-- url: https://github.com/bihealth/annonars-data-clinvar/releases/download/clinvar-weekly-20230625/clinvar-strucvar-grch38-2023-0625+0.6.3.tar.gz
-  excerpt_strategy:
-    strategy: no-excerpt
+    strategy: manual
     count: null
 
 - url: https://github.com/bihealth/mehari-data-tx/releases/download/v0.4.4/mehari-data-txs-grch37-0.4.4.bin.zst

diff --git a/environment.yml b/environment.yml
@@ -46,7 +46,7 @@ dependencies:
   - annonars =0.34.0
   - viguno =0.2.0
   - mehari =0.21.1
-  - varfish-server-worker =0.10.2
+  - varfish-server-worker =0.12.0
   # S3 uploads
   - s5cmd =2.1.0
   # async HTTP requests

diff --git a/excerpt-data/05e93e6f1f5d60e6/gnomad.v4.0.sv.chr4.vcf.gz b/excerpt-data/05e93e6f1f5d60e6/gnomad.v4.0.sv.chr4.vcf.gz
diff --git a/excerpt-data/05e93e6f1f5d60e6/gnomad.v4.0.sv.chr4.vcf.gz.tbi b/excerpt-data/05e93e6f1f5d60e6/gnomad.v4.0.sv.chr4.vcf.gz.tbi
diff --git a/excerpt-data/0be9b2561c9397f2/gnomad.v4.0.sv.chr15.vcf.gz b/excerpt-data/0be9b2561c9397f2/gnomad.v4.0.sv.chr15.vcf.gz
diff --git a/excerpt-data/0be9b2561c9397f2/gnomad.v4.0.sv.chr15.vcf.gz.tbi b/excerpt-data/0be9b2561c9397f2/gnomad.v4.0.sv.chr15.vcf.gz.tbi
diff --git a/excerpt-data/13d4b1406e769b80/gnomad.v4.0.sv.chr3.vcf.gz b/excerpt-data/13d4b1406e769b80/gnomad.v4.0.sv.chr3.vcf.gz
diff --git a/excerpt-data/13d4b1406e769b80/gnomad.v4.0.sv.chr3.vcf.gz.tbi b/excerpt-data/13d4b1406e769b80/gnomad.v4.0.sv.chr3.vcf.gz.tbi
diff --git a/excerpt-data/182806147755e799/gnomad.v4.0.sv.chr5.vcf.gz b/excerpt-data/182806147755e799/gnomad.v4.0.sv.chr5.vcf.gz
diff --git a/excerpt-data/182806147755e799/gnomad.v4.0.sv.chr5.vcf.gz.tbi b/excerpt-data/182806147755e799/gnomad.v4.0.sv.chr5.vcf.gz.tbi
diff --git a/excerpt-data/19a12d845df10514/gnomad.v4.0.sv.chr18.vcf.gz b/excerpt-data/19a12d845df10514/gnomad.v4.0.sv.chr18.vcf.gz
diff --git a/excerpt-data/19a12d845df10514/gnomad.v4.0.sv.chr18.vcf.gz.tbi b/excerpt-data/19a12d845df10514/gnomad.v4.0.sv.chr18.vcf.gz.tbi
diff --git a/excerpt-data/2245e9d640b3f206/gnomad.v4.0.sv.chrX.vcf.gz b/excerpt-data/2245e9d640b3f206/gnomad.v4.0.sv.chrX.vcf.gz
diff --git a/excerpt-data/2245e9d640b3f206/gnomad.v4.0.sv.chrX.vcf.gz.tbi b/excerpt-data/2245e9d640b3f206/gnomad.v4.0.sv.chrX.vcf.gz.tbi
diff --git a/excerpt-data/2c4af2ee68c51be6/clinvar-strucvar-grch38-2023-0625+0.6.3.tar.gz b/excerpt-data/2c4af2ee68c51be6/clinvar-strucvar-grch38-2023-0625+0.6.3.tar.gz
diff --git a/excerpt-data/2c4af2ee68c51be6/url.txt b/excerpt-data/2c4af2ee68c51be6/url.txt
diff --git a/excerpt-data/3390d9ca9ce97bac/gnomad.v4.0.sv.chr2.vcf.gz b/excerpt-data/3390d9ca9ce97bac/gnomad.v4.0.sv.chr2.vcf.gz
diff --git a/excerpt-data/3390d9ca9ce97bac/gnomad.v4.0.sv.chr2.vcf.gz.tbi b/excerpt-data/3390d9ca9ce97bac/gnomad.v4.0.sv.chr2.vcf.gz.tbi
diff --git a/excerpt-data/3b19a5df654e6b4f/gnomad.v4.0.sv.chrY.vcf.gz b/excerpt-data/3b19a5df654e6b4f/gnomad.v4.0.sv.chrY.vcf.gz
diff --git a/excerpt-data/3b19a5df654e6b4f/gnomad.v4.0.sv.chrY.vcf.gz.tbi b/excerpt-data/3b19a5df654e6b4f/gnomad.v4.0.sv.chrY.vcf.gz.tbi
diff --git a/excerpt-data/41e368687b2c69e4/gnomad.v4.0.sv.chr7.vcf.gz b/excerpt-data/41e368687b2c69e4/gnomad.v4.0.sv.chr7.vcf.gz
diff --git a/excerpt-data/41e368687b2c69e4/gnomad.v4.0.sv.chr7.vcf.gz.tbi b/excerpt-data/41e368687b2c69e4/gnomad.v4.0.sv.chr7.vcf.gz.tbi
diff --git a/excerpt-data/555fd2fa82d1b73d/gnomad.v4.0.sv.chr11.vcf.gz b/excerpt-data/555fd2fa82d1b73d/gnomad.v4.0.sv.chr11.vcf.gz
diff --git a/excerpt-data/555fd2fa82d1b73d/gnomad.v4.0.sv.chr11.vcf.gz.tbi b/excerpt-data/555fd2fa82d1b73d/gnomad.v4.0.sv.chr11.vcf.gz.tbi
diff --git a/excerpt-data/6433022d4690f461/gnomad.v4.0.sv.chr21.vcf.gz b/excerpt-data/6433022d4690f461/gnomad.v4.0.sv.chr21.vcf.gz
diff --git a/excerpt-data/6433022d4690f461/gnomad.v4.0.sv.chr21.vcf.gz.tbi b/excerpt-data/6433022d4690f461/gnomad.v4.0.sv.chr21.vcf.gz.tbi
diff --git a/excerpt-data/6f337fba39cc4c2e/gnomad.v4.0.sv.chr13.vcf.gz b/excerpt-data/6f337fba39cc4c2e/gnomad.v4.0.sv.chr13.vcf.gz
diff --git a/excerpt-data/6f337fba39cc4c2e/gnomad.v4.0.sv.chr13.vcf.gz.tbi b/excerpt-data/6f337fba39cc4c2e/gnomad.v4.0.sv.chr13.vcf.gz.tbi
diff --git a/excerpt-data/9cc0f2ee73775510/gnomad.v4.0.sv.chr10.vcf.gz b/excerpt-data/9cc0f2ee73775510/gnomad.v4.0.sv.chr10.vcf.gz
diff --git a/excerpt-data/9cc0f2ee73775510/gnomad.v4.0.sv.chr10.vcf.gz.tbi b/excerpt-data/9cc0f2ee73775510/gnomad.v4.0.sv.chr10.vcf.gz.tbi
diff --git a/excerpt-data/a271968c7bf845c0/gnomad.v4.0.sv.chr16.vcf.gz b/excerpt-data/a271968c7bf845c0/gnomad.v4.0.sv.chr16.vcf.gz
diff --git a/excerpt-data/a271968c7bf845c0/gnomad.v4.0.sv.chr16.vcf.gz.tbi b/excerpt-data/a271968c7bf845c0/gnomad.v4.0.sv.chr16.vcf.gz.tbi
diff --git a/excerpt-data/a83da02295a7ae02/gnomad.v4.0.sv.chr20.vcf.gz b/excerpt-data/a83da02295a7ae02/gnomad.v4.0.sv.chr20.vcf.gz
diff --git a/excerpt-data/a83da02295a7ae02/gnomad.v4.0.sv.chr20.vcf.gz.tbi b/excerpt-data/a83da02295a7ae02/gnomad.v4.0.sv.chr20.vcf.gz.tbi
diff --git a/excerpt-data/aea53328ecd0b712/gnomad.v4.0.sv.chr6.vcf.gz b/excerpt-data/aea53328ecd0b712/gnomad.v4.0.sv.chr6.vcf.gz
diff --git a/excerpt-data/aea53328ecd0b712/gnomad.v4.0.sv.chr6.vcf.gz.tbi b/excerpt-data/aea53328ecd0b712/gnomad.v4.0.sv.chr6.vcf.gz.tbi
diff --git a/excerpt-data/bc6f1860df305e83/gnomad.v4.0.sv.chr17.vcf.gz b/excerpt-data/bc6f1860df305e83/gnomad.v4.0.sv.chr17.vcf.gz
diff --git a/excerpt-data/bc6f1860df305e83/gnomad.v4.0.sv.chr17.vcf.gz.tbi b/excerpt-data/bc6f1860df305e83/gnomad.v4.0.sv.chr17.vcf.gz.tbi
diff --git a/excerpt-data/bdc69c1e4cafdfaa/clinvar-data-extract-vars-20240612+0.17.0.tar.gz b/excerpt-data/bdc69c1e4cafdfaa/clinvar-data-extract-vars-20240612+0.17.0.tar.gz
diff --git a/excerpt-data/bdc69c1e4cafdfaa/url.txt b/excerpt-data/bdc69c1e4cafdfaa/url.txt
diff --git a/excerpt-data/c3ce41a42c6319e0/gnomad.v4.0.sv.chr22.vcf.gz b/excerpt-data/c3ce41a42c6319e0/gnomad.v4.0.sv.chr22.vcf.gz