Merge pull request #22 from clinical-genomics-uppsala/develop

chore: dev to main

.github/workflows/lint.yaml

@@ -17,13 +17,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Python 3.8
+      - name: Set up Python 3.9
         uses: actions/setup-python@v3
         with:
-          python-version: 3.8
-      - name: Add conda to system path
-        run: |
-          echo $CONDA/bin >> $GITHUB_PATH
+          python-version: 3.9
       - name: Install requirements.txt
         run: |
           pip install -r requirements.txt

.github/workflows/pycodestyl.yaml

@@ -21,10 +21,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Set up Python 3.8
+      - name: Set up Python 3.9
         uses: actions/setup-python@v2
         with:
-          python-version: 3.8
+          python-version: 3.9
       - name: Install requirements.txt
         run: |
           pip install -r requirements.txt

.github/workflows/snakemake-dry-run.yaml

@@ -1,8 +1,6 @@
 name: Snakemake dry run
 
-# Controls when the action will run.
 on:
-   # Triggers the workflow on push or pull request events but only for the master and develop branch
   push:
     branches:
      - develop
@@ -11,34 +9,29 @@ on:
     branches:
      - develop
      - main
-
-  # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 
 jobs:
   snakemake-dry-run:
-    name: Run snakemake dry run
+    name: Run snakemake dry run 
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Python 3.8
+      - name: Set up Python 3.9
         uses: actions/setup-python@v3
         with:
-          python-version: 3.8
-      - name: Add conda to system path
-        run: |
-          echo $CONDA/bin >> $GITHUB_PATH
-      - name: Install dependencies
-        run: |
-          conda install -c conda-forge mamba
+          python-version: 3.9
       - name: Install requirements.txt
         run: |
           pip install -r requirements.txt
       - name: Install requirements.test.txt
         run: |
           pip install -r requirements.test.txt
-      - name: snakemake dry run
+      - name: snakemake dry run cpu mode
+        working-directory: .tests/integration
+        run: |
+          snakemake -n -s ../../workflow/Snakefile --configfile config/config.yaml --config aligner=bwa_cpu snp_caller=deepvariant_cpu
+      - name: snakemake dry run gpu mode
         working-directory: .tests/integration
         run: |
-          mamba install -c conda-forge -c bioconda snakemake singularity
-          snakemake -n -s ../../workflow/Snakefile --configfiles config/config.yaml
+          snakemake -n -s ../../workflow/Snakefile --configfile config/config.yaml --config aligner=bwa_gpu snp_caller=deepvariant_gpu

.tests/integration/config/config.yaml

@@ -7,19 +7,19 @@ units: "config/units.tsv"
 
 output: "config/output_list.json"
 
-default_container: "docker://hydragenetics/common:0.3.0"
+default_container: "docker://hydragenetics/common:1.11.1"
 
 modules:
-  alignment: "v0.4.0"
+  alignment: "v0.6.0"
   annotation: "v0.3.0"
   compression: "v2.0.0"
-  filtering: "v0.2.0"
   cnv_sv: "v0.4.1"
+  filtering: "v0.2.0"
+  misc: "v0.2.0"
   parabricks: "v1.1.0"
   prealignment: "v1.1.0"
   qc: "v0.4.1"
-  snv_indels: "v0.5.0"
-
+  snv_indels: "v1.0.0"
 
 reference:
   coverage_bed: "reference/refseq_select_mane_20221123.bed"   #Created bed with https://github.com/zezzipa/UCSCtable2bed
@@ -28,13 +28,16 @@ reference:
   fasta: "reference/homo_sapiens.fasta"
   fai: "reference/homo_sapiens.fasta.fai"
   genepanels: "reference/genepanels.list"
-  genome_dir: "reference/homo_sapiens_star"
   sites: "reference/homo_sapiens.known_indels.vcf.gz"
+  skip_chrs:
+    - chrM
+  skip_contigs:
+    - chrM
 
 trimmer_software: "fastp_pe"
 
-aligner: "bwa_gpu" # or "bwa_cpu"
-snp_caller: "deepvariant_gpu" # or "deepvariant_cpu"
+aligner: "bwa_cpu" # or "bwa_cpu"
+snp_caller: "deepvariant_cpu" # or "deepvariant_cpu"
 
 automap:
   container: "docker://hydragenetics/automap:1.2"
@@ -56,32 +59,11 @@ bwa_mem:
 create_cov_excel:
   covLimits: "10 20 30"
 
-deeptrio_call_variants:
-  container: "docker://google/deepvariant:deeptrio-1.4.0-gpu"
-  model:
-    child: "/opt/models/deeptrio/wes/child/model.ckpt" # path to model in the deepvariant container
-    parent: "/opt/models/deeptrio/wes/parent/model.ckpt"
-
-deeptrio_make_examples:
-  container: "docker://google/deepvariant:deeptrio-1.4.0" #"docker://google/deepvariant:deeptrio-1.4.0-gpu"
-  extra: "--channels insert_size --pileup_image_height_child 100 --pileup_image_height_parent 100 --regions /beegfs-storage/projects/wp3/Reference_files/Manifest/Clinical_research_exome/TWIST/Twist_Comprehensive_Exome_Covered_Targets_hg38_20bp_pad.bed"
-  n_shards: 80
-
-deeptrio_postprocess_variants:
-  container: "docker://google/deepvariant:deeptrio-1.4.0"
-
-deepvariant_call_variants:
-  container: "docker://hydragenetics/deepvariant:1.4.0" #"docker://hydragenetics/deepvariant:1.4.0"
-  model: "/opt/models/wes/model.ckpt" # path to wes model in the deepvariant container
-
-deepvariant_make_examples:
-  container: "docker://hydragenetics/deepvariant:1.4.0"
-  extra: "--channels insert_size --regions /beegfs-storage/projects/wp3/Reference_files/Manifest/Clinical_research_exome/TWIST/Twist_Comprehensive_Exome_Covered_Targets_hg38_20bp_pad.bed"
-  n_shards: 80
-
-deepvariant_postprocess_variants:
+deepvariant:
   container: "docker://hydragenetics/deepvariant:1.4.0"
-  vcf_type: "gvcf"
+  bed: "reference/Twist_Comprehensive_Exome_Covered_Targets_hg38_20bp_pad.bed"
+  model_type: "WES"
+  output_gvcf: True
 
 exomedepth_call:
   container: "docker://hydragenetics/exomedepth:1.1.15"
@@ -103,12 +85,11 @@ fastp_pe:
 fastqc:
   container: "docker://hydragenetics/fastqc:0.11.9"
 
-glnexus:
+glnexus_peddy:
   container: "docker://ghcr.io/dnanexus-rnd/glnexus:v1.4.1"
   configfile: "DeepVariantWES"
-  extra: "--bed reference/Twist_Comprehensive_Exome_Covered_Targets_hg38_with_names.bed"
 
-glnexus_peddy:
+glnexus_trio:
   container: "docker://ghcr.io/dnanexus-rnd/glnexus:v1.4.1"
   configfile: "DeepVariantWES"
 
@@ -126,7 +107,16 @@ multiqc:
         - "qc/fastqc/{sample}_{type}_{flowcell}_{lane}_{barcode}_fastq1_fastqc.zip"
         - "qc/fastqc/{sample}_{type}_{flowcell}_{lane}_{barcode}_fastq2_fastqc.zip"
         - "qc/mosdepth_bed/{sample}_{type}.mosdepth.summary.txt"
-        - "qc/mosdepth_bed/{sample}_{type}.per-base.bed.gz"
+        - "qc/mosdepth_bed_design/{sample}_{type}.mosdepth.summary.txt"
+        - "qc/mosdepth_bed_design/{sample}_{type}.per-base.bed.gz"
+        - "qc/mosdepth_bed_design/{sample}_{type}.regions.bed.gz"
+        - "qc/mosdepth_bed_design/{sample}_{type}.mosdepth.region.dist.txt"
+        - "qc/mosdepth_bed_design/{sample}_{type}.mosdepth.global.dist.txt"
+        - "qc/peddy/peddy.peddy.ped"
+        - "qc/peddy/peddy.background_pca.json"
+        - "qc/peddy/peddy.ped_check.csv"
+        - "qc/peddy/peddy.sex_check.csv"
+        - "qc/peddy/peddy.het_check.csv"
         - "qc/peddy/peddy_sex_check_mqc.tsv"
         - "qc/peddy/peddy_rel_check_mqc.tsv"
         - "qc/picard_collect_alignment_summary_metrics/{sample}_{type}.alignment_summary_metrics.txt"
@@ -179,19 +169,17 @@ sample_order_for_multiqc:
   sample_sheet: "config/SampleSheet.csv"
 
 spring:
-  container: "docker://hydragenetics/spring:1.0.1"
+  container: "docker://hydragenetics/spring:1.1.1"
 
-tabix:
-  container: "docker://hydragenetics/common:0.1.0"
 
 upd:
   container: "docker://hydragenetics/upd:0.1.1"
   extra: "--vep "
 
-vep:
+vep_trio:
   container: "docker://ensemblorg/ensembl-vep:release_109.3"
-  vep_cache: "refereence/VEP/"
-  extra: "--assembly GRCh38 --max_af"
+  vep_cache: "reference/VEP/"
+  extra: "--assembly GRCh38 --check_existing --pick --max_af --buffer_size 50000"
 
 vt_decompose:
   container: "docker://hydragenetics/vt:2015.11.10"

.tests/integration/config/output_list.json

@@ -10,5 +10,6 @@
   "results/{sample}/{sample}_N.cram": {"name": "_copy_samtools_cram", "file": "compression/samtools_view/{sample}_N.cram", "types": ["N"]},
   "results/{sample}/{sample}_N.cram.crai": {"name": "_copy_samtools_crai", "file": "compression/samtools_view/{sample}_N.cram.crai", "types": ["N"]},
   "results/{sample}/spring/{sample}_{type}_{flowcell}_{lane}_{barcode}.spring":  {"name": "_copy_spring", "file": "compression/spring/{sample}_{type}_{flowcell}_{lane}_{barcode}.spring", "types": ["N"]},
+  "results/peddy.html": {"name": "_copy_peddy_html", "file": "qc/peddy/peddy.html", "types": ["N"]},
   "results/multiqc_DNA.html": {"name": "_copy_multiqc_html", "file": "qc/multiqc/multiqc_DNA.html", "types": ["N"]}
 }

.tests/integration/config/sample_order.tsv

@@ -1,5 +1,7 @@
 Sample Order	Sample Name
-sample_001	D20-06511
-sample_002	D20-06203
-sample_003	NA12878
-sample_004	NA18507
+sample_001	NA12878
+sample_002	NA12911
+sample_003	HG002
+sample_004	HG003
+sample_005	HG004
+sample_006	sample6

.tests/integration/config/sample_replacement.tsv

@@ -1,2 +1,6 @@
-NA12878	sample_003
-NA18507	sample_004
+NA12878	sample_001
+NA12911	sample_002
+HG002	sample_003
+HG003	sample_004
+HG004	sample_005
+sample6	sample_006

.tests/integration/config/samples.tsv

@@ -1,3 +1,7 @@
 sample	tumor_content	sex	trioid	trio_member
 NA12878	0.0	female	NA	NA
 NA12911	0.0	male	NA	NA
+HG002	0.0	male	Trio1	proband
+HG003	0.0	male	Trio1	father
+HG004	0.0	male	Trio1	mother
+sample5	0.0	female	Trio1	proband

.tests/integration/config/units.tsv

@@ -1,3 +1,7 @@
 sample	type	platform	barcode	machine	flowcell	lane	fastq1	fastq2	adapter
 NA12878	N	Illumina	CAAGCTAG+ACATAGCG	@A00687	HVWHGDSXX	L001	fastq/NA12878.1_1.fastq.gz	fastq/NA12878.1_2.fastq.gz	AGATCGGAAGAGCACACGTCTGAACTCCAGTCA,AGATCGGAAGAGCGTCGTGTAGGGAAAGAGTGT
 NA12911	N	Illumina	CAAGCTAG+ACATAGCG	@A00687	HVWHGDSXX	L001	fastq/NA12911.1_1.fastq.gz	fastq/NA12911.1_2.fastq.gz	AGATCGGAAGAGCACACGTCTGAACTCCAGTCA,AGATCGGAAGAGCGTCGTGTAGGGAAAGAGTGT
+HG002	N	Illumina	CAAGCTAG+ACATAGCG	@A00687	HVWHGDSXX	L001	fastq/HG002.1_1.fastq.gz	fastq/HG002.1_2.fastq.gz	AGATCGGAAGAGCACACGTCTGAACTCCAGTCA,AGATCGGAAGAGCGTCGTGTAGGGAAAGAGTGT
+HG003	N	Illumina	CAAGCTAG+ACATAGCG	@A00687	HVWHGDSXX	L001	fastq/HG003.1_1.fastq.gz	fastq/HG003.1_2.fastq.gz	AGATCGGAAGAGCACACGTCTGAACTCCAGTCA,AGATCGGAAGAGCGTCGTGTAGGGAAAGAGTGT
+HG004	N	Illumina	CAAGCTAG+ACATAGCG	@A00687	HVWHGDSXX	L001	fastq/HG004.1_1.fastq.gz	fastq/HG004.1_2.fastq.gz	AGATCGGAAGAGCACACGTCTGAACTCCAGTCA,AGATCGGAAGAGCGTCGTGTAGGGAAAGAGTGT
+sample5	N	Illumina	CAAGCTAG+ACATAGCG	@A00687	HVWHGDSXX	L001	fastq/sample5.1_1.fastq.gz	fastq/sample5.1_2.fastq.gz	AGATCGGAAGAGCACACGTCTGAACTCCAGTCA,AGATCGGAAGAGCGTCGTGTAGGGAAAGAGTGT

.tests/integration/reference/homo_sapiens.fasta.fai

@@ -0,0 +1,25 @@
+chr1	248956422	6	70	71
+chr2	242193529	252512955	70	71
+chr3	198295559	498166398	70	71
+chr4	190214555	699294757	70	71
+chr5	181538259	892226669	70	71
+chr6	170805979	1076358338	70	71
+chr7	159345973	1249604409	70	71
+chr8	145138636	1411226760	70	71
+chr9	138394717	1558438812	70	71
+chr10	133797422	1698810604	70	71
+chr11	135086622	1834519425	70	71
+chr12	133275309	1971535863	70	71
+chr13	114364328	2106715112	70	71
+chr14	107043718	2222713224	70	71
+chr15	101991189	2331286145	70	71
+chr16	90338345	2434734358	70	71
+chr17	83257441	2526363258	70	71
+chr18	80373285	2610810099	70	71
+chr19	58617616	2692331581	70	71
+chr20	64444167	2751786599	70	71
+chr21	46709983	2817151404	70	71
+chr22	50818468	2864528680	70	71
+chrX	156040895	2916073133	70	71
+chrY	57227415	3074343190	70	71
+chrM	16569	3132388146	70	71