Merge pull request #54 from clinical-genomics-uppsala/novaseq_x_valid…

…ation ci: update config files and create new marvin profiles
clinical-genomics-uppsala · Apr 18, 2024 · 7bfa116 · 7bfa116
2 parents a780bee + 44857c8
commit 7bfa116
Show file tree

Hide file tree

Showing 69 changed files with 6,409 additions and 1,977 deletions.
diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml
@@ -17,13 +17,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Python 3.8
+      - name: Set up Python 3.9
         uses: actions/setup-python@v3
         with:
-          python-version: 3.8
-      - name: Add conda to system path
-        run: |
-          echo $CONDA/bin >> $GITHUB_PATH
+          python-version: 3.9
       - name: Install requirements.txt
         run: |
           pip install -r requirements.txt

diff --git a/.github/workflows/pycodestyle.yaml b/.github/workflows/pycodestyle.yaml
@@ -17,10 +17,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Python 3.8
+      - name: Set up Python 3.9
         uses: actions/setup-python@v3
         with:
-          python-version: 3.8
+          python-version: 3.9
       - name: Install requirements.txt
         run: |
           pip install -r requirements.txt

diff --git a/.github/workflows/snakemake-dry-run_cpu.yaml b/.github/workflows/snakemake-dry-run_cpu.yaml
@@ -0,0 +1,33 @@
+name: Snakemake dry run
+
+on:
+  push:
+    branches:
+     - develop
+     - main
+  pull_request:
+    branches:
+     - develop
+     - main
+  workflow_dispatch:
+
+jobs:
+  snakemake-dry-run:
+    name: Run snakemake dry run for cpu mode
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v3
+        with:
+          python-version: 3.9
+      - name: Install requirements.txt
+        run: |
+          pip install -r requirements.txt
+      - name: Install requirements.test.txt
+        run: |
+          pip install -r requirements.test.txt
+      - name: snakemake dry run cpu mode
+        working-directory: .tests/integration
+        run: |
+          snakemake -n -s ../../workflow/Snakefile --configfile config/config.yaml --config aligner=bwa_cpu --config snp_caller=deepvariant_cpu
diff --git a/.github/workflows/snakemake-dry-run.yaml → .github/workflows/snakemake-dry-run_gpu.yaml b/.github/workflows/snakemake-dry-run.yaml → .github/workflows/snakemake-dry-run_gpu.yaml
@@ -13,21 +13,21 @@ on:
 
 jobs:
   snakemake-dry-run:
-    name: Run snakemake dry run
+    name: Run snakemake dry run for gpu mode
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Python 3.8
+      - name: Set up Python 3.9
         uses: actions/setup-python@v3
         with:
-          python-version: 3.8
+          python-version: 3.9
       - name: Install requirements.txt
         run: |
           pip install -r requirements.txt
       - name: Install requirements.test.txt
         run: |
           pip install -r requirements.test.txt
-      - name: snakemake dry run
+      - name: snakemake dry run gpu mode
         working-directory: .tests/integration
         run: |
           snakemake -n -s ../../workflow/Snakefile --configfile config/config.yaml
diff --git a/.github/workflows/test-build-mkdocs b/.github/workflows/test-build-mkdocs
@@ -0,0 +1,33 @@
+name: build mkdocs
+
+on:
+  push:
+    branches:
+     - develop
+     - master
+  pull_request:
+    branches:
+     - develop
+     - master
+  workflow_dispatch:
+
+jobs:
+  build-mkdocs:
+    name: build mkdocs
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v3
+        with:
+          python-version: 3.8
+      - name: Install requirements.txt
+        run: |
+          pip install -r requirements.txt
+      - name: Install requirements.test.txt
+        run: |
+          pip install -r requirements.test.txt
+          pip install -r docs/requirements.txt
+      - name: build mkdocs
+        run: |
+          mkdocs build
diff --git a/.readthedocs.yaml b/.readthedocs.yaml
@@ -0,0 +1,19 @@
+# Read the Docs configuration file for MkDocs projects
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the version of Python and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.12"
+
+mkdocs:
+  configuration: mkdocs.yml
+
+# Optionally declare the Python requirements required to build your docs
+python:
+  install:
+  - requirements: docs/requirements.txt
diff --git a/.tests/integration/config/config.yaml b/.tests/integration/config/config.yaml
@@ -10,16 +10,17 @@ output: "config/output_list.json"
 default_container: "docker://hydragenetics/common:0.1.9"
 
 modules:
-  alignment: "v0.4.0"
+  alignment: "v0.6.0"
   annotation: "v0.3.0"
   compression: "v2.0.0"
-  cnv_sv: "v0.4.1"
-  filtering: "v0.2.0"
-  parabricks: "v1.1.0"
-  prealignment: "v1.1.0"
-  qc: "da66130"
+  cnv_sv: "v0.5.0"
+  filtering: "v0.3.0"
+  parabricks: "v1.2.0"
+  prealignment: "v1.2.0"
+  qc: "v0.5.0"
+  misc: "v0.2.0"
   mitochondrial: "v0.1.0"
-  snv_indels: "v0.5.0"
+  snv_indels: "v1.0.0"
 
 reference:
   coverage_bed: "reference/refseq_select_mane_20221123.bed"   #Created bed with https://github.com/zezzipa/UCSCtable2bed
@@ -31,26 +32,39 @@ reference:
   genepanels: "reference/genepanels.list"
   sites: "reference/homo_sapiens.known_indels.vcf.gz"
   wgs_intervals: "reference/homo_sapiens.wgs.interval_list"
+  merge_contigs: # contigs to be merged to a single BAM for mark duplicates
+    - ".*_random"
+    - "chrUn_.*"
+    - "chrEBV"
 
 trimmer_software: "fastp_pe"
 
-aligner: "bwa_cpu" # "bwa_cpu"
-snp_caller: "deepvariant_cpu" # "deepvariant_cpu"
+aligner: "bwa_gpu" # "bwa_cpu"
+snp_caller: "deepvariant_gpu" # "deepvariant_cpu"
 
 automap:
   container: "docker://hydragenetics/automap:1.2"
   build: "hg38"
   extra: "--DP 10 --minsize 3 --chrX"
   outdir: "cnv_sv/automap"
 
-bcftools_view_pass:
-  container: "docker://hydragenetics/common:0.3.0"
-  extra: "--apply-filter PASS "
+bcftools_split_vep:
+  columns: "gnomad_AF:Float"
+
+bcftools_view_biallelic:
+  extra: "-m2 -M2 -v snps"
+
+bcftools_filter_cnvpytor:
+  exclude: "'pytorp1>0.0001 | pytorDG<100000 | pytorpN>0.5 | pytorQ0>0.5'"
+  extra: "-s LowQual "
 
 bcftools_view_pass:
-  container: "docker://hydragenetics/common:0.3.0"
+  container: "docker://hydragenetics/common:1.11.1"
   extra: "--apply-filter PASS "
 
+bcftools_view_svdb:
+  extra: "-f 'PASS' -e 'tiddit_AF>0.05 | manta_AF>0.05 | cnvpytor_AF>0.05 | gnomad_AF>0.1'"
+
 bwa_mem:
   container: "docker://hydragenetics/bwa:0.7.15"
   amb: "reference/homo_sapiens.fasta.amb"
@@ -81,31 +95,25 @@ bwa_mem_mito:
 cnvpytor_readdepth:
   container: "docker://hydragenetics/cnvpytor:1.2.1"
   length_list: "1000 10000 100000"
+  extra: "-chrom chr1 chr2 chr3 chr4 chr5 chr6 chr7 chr8 chr9 chr10 chr11 chr12 chr13 chr14 chr15 chr16 chr17 chr18 chr19 chr20 chr21 chr22 chrX chrY chrM "
 
 cnvpytor_filter:
-  container: "docker://hydragenetics/cnvpytor:1.2.1"
+  container: "docker://hydragenetics/cnvpytor:1.3.1"
   dG_range: "100000 inf"
-  p_range: "0 0.01"
+  calling_model: "rd_mean_shift"
+  p_range: "0 0.0001"
   pN_range: "0 0.5"
-  Q0_range: "0 0.05"
-  view: "1000"
+  Q0_range: "-1 0.5"
+  view: "500"
 
 create_cov_excel:
-  container: "docker://hydragenetics/common:0.3.0"
+  container: "docker://hydragenetics/common:1.11.1"
   covLimits: "10 20 30"
 
-deepvariant_call_variants:
-  container: "docker://hydragenetics/deepvariant:1.4.0" #"docker://hydragenetics/deepvariant:1.4.0"
-  model: "/opt/models/wes/model.ckpt" # path to model in the deepvariant container
-
-deepvariant_make_examples:
-  container: "docker://hydragenetics/deepvariant:1.4.0"
-  extra: "--channels insert_size"
-  n_shards: 80
-
-deepvariant_postprocess_variants:
-  container: "docker://hydragenetics/deepvariant:1.4.0"
-  vcf_type: "gvcf"
+deepvariant:
+  container: "docker://google/deepvariant:1.5.0" 
+  model_type: "WGS" 
+  output_gvcf: true
 
 expansionhunter:
   container: "docker://hydragenetics/expansionhunter:5.0.0"
@@ -181,9 +189,9 @@ gatk_split_multi_allelic_sites:
 gatk_variant_filtration:
   container: "docker://hydragenetics/gatk4:4.2.2.0"
 
-glnexus:
-  bedfile: "reference/GRCH38.bed"
-  container: "docker://ghcr.io/dnanexus-rnd/glnexus:v1.4.3"
+glnexus_trio:
+  container: "docker://ghcr.io/dnanexus-rnd/glnexus:v1.4.1"
+  configfile: "DeepVariantWGS"
 
 glnexus_peddy:
   container: "docker://ghcr.io/dnanexus-rnd/glnexus:v1.4.1"
@@ -195,7 +203,7 @@ haplocheck:
 
 manta_config_n:
   container: "docker://hydragenetics/manta:1.6.0"
-  extra: ""
+  extra: "--callRegions reference/GRCh38_chromosomes.bed.gz"
 
 manta_run_workflow_n:
   container: "docker://hydragenetics/manta:1.6.0"
@@ -236,19 +244,22 @@ multiqc:
         - "qc/picard_collect_wgs_metrics/{sample}_{type}.txt"
         - "qc/samtools_stats/{sample}_{type}.samtools-stats.txt"
         - "qc/samtools_idxstats/{sample}_{type}.samtools-idxstats.txt"
+        - "qc/verifybamid2/{sample}_{type}.selfSM"
 
 mt_reference:
   mt: "reference/Homo_sapiens_assembly38.chrM.fasta"
   mt_shifted: "reference/Homo_sapiens_assembly38.chrM.shifted_by_8000_bases.fasta"
   blacklist: reference/blacklist_sites.hg38.chrM.bed
 
 pbrun_deepvariant:
-  container: "docker://nvcr.io/nvidia/clara/clara-parabricks:4.0.0-1"
-  extra: "--pb-model-file reference/deepvariant.eng --disable-use-window-selector-model --gvcf" ## For consistency with deepvariant cpu. Also increases accuracy.
+  container: "docker://nvcr.io/nvidia/clara/clara-parabricks:4.2.1-1"
+  extra: "--num-cpu-threads-per-stream 5 --num-streams-per-gpu 2 --disable-use-window-selector-model --gvcf " ## For consistency with deepvariant cpu. Also increases accuracy. 
+  tmp_dir: "/tmp/"
 
 pbrun_fq2bam:
-  container: "docker://nvcr.io/nvidia/clara/clara-parabricks:4.0.0-1"
-  extra: ""
+  container: "docker://nvcr.io/nvidia/clara/clara-parabricks:4.2.1-1"
+  extra: "--gpusort --gpuwrite --num-cpu-threads-per-stage 10 "
+  tmp_dir: "/tmp/"
 
 peddy:
   container: "docker://hydragenetics/peddy:0.4.8"
@@ -280,13 +291,13 @@ reviewer:
   container: "docker://hydragenetics/reviewer:0.2.7"
 
 samtools_stats:
-  container: "docker://hydragenetics/common:0.3.0"
+  container: "docker://hydragenetics/common:1.11.1"
 
 samtools_idxstats:
-  container: "docker://hydragenetics/common:0.3.0"
+  container: "docker://hydragenetics/common:1.11.1"
 
 samtools:
-  container: "docker://hydragenetics/common:0.3.0"
+  container: "docker://hydragenetics/common:1.11.1"
 
 smn_caller:
   container: "docker://hydragenetics/smncopynumbercaller:1.1.2"
@@ -303,8 +314,10 @@ stranger:
   catalog: "reference/variant_catalog_grch38.json"
 
 svdb_merge:
-  container: "docker://hydragenetics/svdb:2.6.0"
+  container: "docker://hydragenetics/svdb:2.8.2"
   overlap: 0.6
+  priority: "tiddit,manta,cnvpytor"
+  extra: "--pass_only "
 
 svdb_query:
   container: "docker://hydragenetics/svdb:2.6.0"
@@ -332,6 +345,11 @@ vep_trio:
   vep_cache: "reference/VEP/"
   extra: "--assembly GRCh38 --check_existing --pick --max_af --buffer_size 50000"
 
+vep_svdb:
+  container: "docker://ensemblorg/ensembl-vep:release_110.1"
+  vep_cache: "reference/VEP/"
+  extra: "--assembly GRCh38 --check_existing --pick --custom file=/data/ref_data/gnomad/gnomad_sv/gnomad.v4.0.sv.vcf.gz,short_name=gnomad,fields=AF%CN_NONREF_FREQ%FILTER%ALGORITHMS%EVIDENCE%SVTYPE,format=vcf,reciprocal=1,overlap_cutoff=60,distance=1000,same_type=1"
+
 verifybamid2:
   container: "docker://hydragenetics/verifybamid2:2.0.1"
   svd_mu: "reference/1000g.phase3.100k.b38.vcf.gz.dat.mu"

diff --git a/.tests/integration/config/output_list.json b/.tests/integration/config/output_list.json
@@ -4,12 +4,13 @@
   "results/{sample}/{sample}_snv_indels.filtered.vcf.gz": {"name": "_copy_snv_indels_filtered_vcf", "file": "vcf_final/{sample}_N.vep_annotated.filter.germline.vcf.gz", "types": ["N"]},
   "results/{sample}/{sample}_snv_indels.filtered.vcf.gz.tbi": {"name": "_copy_snv_indels_filtered_tbi", "file": "vcf_final/{sample}_N.vep_annotated.filter.germline.vcf.gz.tbi", "types": ["N"]},
   "results/{sample}/cnv_sv/{sample}.cnvpytor_filtered.vcf.gz": {"name": "_copy_cnvpytor_filtered_vcf", "file": "cnv_sv/cnvpytor/{sample}_N.filtered.vcf.gz", "types": ["N"]},
-  "results/{sample}/cnv_sv/{sample}.cnvpytor.vcf.gz": {"name": "_copy_cnvpytor_vcf", "file": "cnv_sv/cnvpytor/{sample}_N.vcf.gz", "types": ["N"]},
+  "results/{sample}/cnv_sv/{sample}.cnvpytor.vcf.gz": {"name": "_copy_cnvpytor_vcf", "file": "cnv_sv/cnvpytor/{sample}_N.softfiltered.vcf.gz", "types": ["N"]},
   "results/{sample}/cnv_sv/{sample}.cnvpytor_filtered.aed": {"name": "_copy_cnvpytor_filtered_aed", "file": "cnv_sv/cnvpytor/{sample}_N_filtered.aed", "types": ["N"]},
   "results/{sample}/cnv_sv/{sample}.cnvpytor.aed": {"name": "_copy_cnvpytor_aed", "file": "cnv_sv/cnvpytor/{sample}_N.aed", "types": ["N"]},
   "results/{sample}/cnv_sv/{sample}.tiddit.vcf.gz": {"name": "_copy_tiddit_vcf", "file": "cnv_sv/tiddit/{sample}_N_ref.vcf.gz", "types": ["N"]},
   "results/{sample}/cnv_sv/{sample}.manta_diploidSV.vcf.gz": {"name": "_copy_manta_vcf", "file": "cnv_sv/manta_run_workflow_n/{sample}/results/variants/diploidSV.vcf.gz", "types": ["N"]},
-  "results/{sample}/cnv_sv/{sample}.svdb_merged.vcf.gz": {"name": "_copy_svdb_merged_vcf", "file": "cnv_sv/svdb_merge/{sample}_N.merged_ref.vcf.gz", "types": ["N"]},
+  "results/{sample}/cnv_sv/{sample}.svdb_merged.vcf.gz": {"name": "_copy_svdb_merged_vcf", "file": "annotate/vep_svdb/{sample}_N.merged.svdb_query.vep_annotated.vcf.gz", "types": ["N"]},
+  "results/{sample}/cnv_sv/{sample}.svdb_merged.filtered.vcf.gz": {"name": "_copy_svdb_merged_filtered_vcf", "file": "annotate/vep_svdb/{sample}_N.merged.svdb_query.vep_annotated.filtered.vcf.gz", "types": ["N"]},
   "results/{sample}/{sample}.upd_regions.bed": {"name": "_copy_upd_regions_bed", "file": "cnv_sv/upd/{sample}_N.upd_regions.bed", "types": ["N"]},
   "results/{sample}/{sample}.contamination.html": {"name": "_copy_haplochack_contamination_report", "file": "mitochondrial/haplocheck/{sample}_N.contamination.html", "types": ["N"]},
   "results/{sample}/{sample}.expansionhunter_stranger.vcf.gz": {"name": "_copy_stranger_vcf", "file": "cnv_sv/stranger/{sample}_N.stranger.vcf.gz", "types": ["N"]},

diff --git a/.tests/integration/config/SampleSheet.csv → ...ation/reference/GRCh38_chromosomes.bed.gz b/.tests/integration/config/SampleSheet.csv → ...ation/reference/GRCh38_chromosomes.bed.gz
diff --git a/snakemake-workflow-catalog.yml → ...egration/reference/homo_sapiens.fasta.amb b/snakemake-workflow-catalog.yml → ...egration/reference/homo_sapiens.fasta.amb
diff --git a/.tests/integration/reference/homo_sapiens.fasta.ann b/.tests/integration/reference/homo_sapiens.fasta.ann
diff --git a/.tests/integration/reference/homo_sapiens.fasta.bwt b/.tests/integration/reference/homo_sapiens.fasta.bwt