feat: everything

.github/CODEOWNERS

@@ -2,4 +2,3 @@
 # the repo. Unless a later match takes precedence,
 # @global-owner1 and @global-owner2 will be requested for
 # review when someone opens a pull request.
-* @marrip

.github/workflows/conventional-prs.yml

@@ -0,0 +1,18 @@
+name: PR
+on:
+  pull_request_target:
+    types:
+      - opened
+      - reopened
+      - edited
+      - synchronize
+
+jobs:
+  title-format:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: amannn/[email protected]
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          validateSingleCommit: true

.github/workflows/lint.yaml

@@ -0,0 +1,33 @@
+name: Lint
+
+on:
+  push:
+    branches:
+     - develop
+     - main
+  pull_request:
+    branches:
+     - develop
+     - main
+  workflow_dispatch:
+
+jobs:
+  lint:
+    name: Lint workflow
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v3
+        with:
+          python-version: 3.8
+      - name: Add conda to system path
+        run: |
+          echo $CONDA/bin >> $GITHUB_PATH
+      - name: Install requirements.txt
+        run: |
+          pip install -r requirements.txt
+      - name: Linting
+        working-directory: .tests/integration
+        run: |
+          snakemake --lint -n -s ../../workflow/Snakefile --configfiles ../../config/config.yaml config.yaml

.github/workflows/pycodestyle.yaml

@@ -0,0 +1,41 @@
+name: pycodestyle
+
+# Controls when the action will run.
+on:
+   # Triggers the workflow on push or pull request events but only for the master and develop branch
+  push:
+    branches:
+     - develop
+     - main
+  pull_request:
+    branches:
+     - develop
+     - main
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  pycodestyle:
+    name: pycodestyle
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+      - name: Add conda to system path
+        run: |
+          echo $CONDA/bin >> $GITHUB_PATH
+      - name: Install dependencies
+        run: |
+          conda install -c conda-forge mamba
+      - name: Install requirements.txt
+        run: |
+          pip install -r requirements.txt
+      - name: Install requirements.test.txt
+        run: |
+          pip install -r requirements.test.txt
+      - name: Check python syntax lib
+        run: pycodestyle --max-line-length=130 --statistics workflow/scripts

.github/workflows/release-please.yaml

@@ -0,0 +1,16 @@
+name: release-please
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  release-please:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: GoogleCloudPlatform/release-please-action@v2
+        id: release
+        with:
+          release-type: python
+          package-name: fluffy_hematology_wgs
+          default-branch: main

.github/workflows/snakefmt.yaml

@@ -21,10 +21,10 @@ jobs:
         with:
           fetch-depth: 0
       - name: Lint Code Base
-        uses: github/super-linter@v4
+        uses: super-linter/super-linter@v5.3.1
         env:
           DEFAULT_BRANCH: main
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          OUTPUT_FOLDER: .github/linters/report
           VALIDATE_ALL_CODEBASE: false
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           VALIDATE_SNAKEMAKE_SNAKEFMT: true
+          OUTPUT_FOLDER: .github/linters/report

.github/workflows/snakemake-dry-run.yaml

@@ -30,4 +30,4 @@ jobs:
       - name: snakemake dry run
         working-directory: .tests/integration
         run: |
-          snakemake -n -s ../../workflow/Snakefile
+          snakemake --configfiles ../../config/config.yaml config.yaml -n -s ../../workflow/Snakefile

.tests/integration/SampleSheet.csv

@@ -0,0 +1,19 @@
+[Header]
+Done at NGI
+Experiment Name,XXX
+Date,230828
+Module,GenerateFASTQ - 2.0.1
+Workflow,GenerateFASTQ
+Library Prep Kit,Custom
+Chemistry,
+[Reads]
+149
+149
+
+[Settings]
+
+[Data]
+Sample_ID,Sample_Name,Description,I7_Index_ID,index,I5_Index_ID,index2,Sample_Project
+D99-0000,D99-0000,T_K_PED99-0000_XXX_PROJ-0001,A7,AAATTTCCCT,A7,ACTACTACTGT,WGSWP2
+D99-0001,D99-0001,N_K_PED99-0000_XXX_PROJ-0001,A8,AAATTCATCT,A8,AGGGTGAACTT,WGSWP2
+D99-0001,D99-0001,R_K_PED99-0000_XXX_PROJ-0001,B7,GGTTTCGCCT,B7,AGATGTGACTT,WGSWP2

.tests/integration/config.yaml

@@ -1,157 +1,119 @@
-resources: "resources.yaml"
-samples: "samples.tsv"
-units: "units.tsv"
-
-modules:
-  annotation: "e58432c3467db31621ab51a16034a026cf2a2c84"
-  cnv_sv: "4b3f69ef0f0781b7943ca27ca16b940470c27118"
-  compression: "v1.1.0"
-  parabricks: "v0.2.0"
-  prealignment: "v0.2.0"
-  qc: "ca81558181c16766674ddc9179e27928e87cc32b"
-  misc: "v0.1.0"
-
-default_container: "docker://hydragenetics/common:0.1.3"
+output: "../../config/output_files.json"
 
 reference:
-  fai: "reference/reference.fasta.fai"
+  design_bed: "reference/homo_sapiens.wgs.bed"
+  design_intervals: "reference/wgs.interval_list" #HSMetrics
+  design_intervals_gatk_cnv: "reference/wgs.interval_list"
   fasta: "reference/reference.fasta"
+  fai: "reference/reference.fasta.fai"
+  fasta_rna: "reference/reference.fasta"
   genome_dir: "reference"
   sites: "reference/sites.vcf.gz"
   wgs_intervals: "reference/wgs.interval_list"
 
-trimmer_software: "fastp_pe"
+annotate_cnv:
+  cna_exons: "reference/hg38_CNA.bed" 
 
-bedtools_intersect:
-  extra: "-header"
+arriba:
+  assembly: "reference/assembly.fasta"
+  blacklist: "reference/arriba_blacklist.txt"
+  extra: "-p abc.gff3 -k abc.tsv.gz"
+  gtf: "reference/assembly.gtf"
 
-cnvkit_batch:
-  container: "docker://marrip/cnvkit:0.9.9"
-  method: "wgs"
-  normal_reference: "reference/cnvkit.cnn"
+arriba_draw_fusion:
+  cytobands: "reference/cytobands.tsv"
+  gtf: "reference/assembly.gtf"
+  protein_domains: "reference/proteindomain.gff3"
 
-cnvkit_diagram:
-  container: "docker://marrip/cnvkit:0.9.9"
+bcftools_SNV:
+  all: "reference/all.bed"
+  aml: "reference/aml.bed"
+  tm : "reference/tm.bed"
 
-cnvkit_call:
-  container: "docker://marrip/cnvkit:0.9.9"
+bcftools_SV:
+  all: "reference/all.bed"
+  aml: "reference/aml.bed"
 
-cnvkit_scatter:
-  container: "docker://marrip/cnvkit:0.9.9"
-  extra: "-c {locus} --y-min -10 --y-max 10"
-
-cnvkit_vcf:
-  container: "docker://python:3.9.9-slim-buster"
+cnvkit_batch:
+  normal_reference: "reference/cnvkit.cnn"
 
-config_manta_tn:
-  container: "docker://marrip/manta:v1.6.0"
+cnvkit_table:
+  bedfile: "reference/cna_genes.bed"
+  cyto_coordinates: "reference/cyto_coordinates.txt"
+  # log_thresholds: "-0.19,0.2"
+  # ploidy: "3"
 
-crumble:
-  container: "docker://hydragenetics/crumble:0.8.3"
-  extra: "-O CRAM"
+cnv_html_report:
+  show_table: true
+  template_dir: "../../config/cnv_report_template"
+  cnv_vcf:
+    - annotation: cna_exons
+      filter: cnv_hardfilter
 
-fastp_pe:
-  container: "docker://hydragenetics/fastp:0.20.1"
 
-fastqc:
-  container: "docker://hydragenetics/fastqc:0.11.9"
+filter_vcf:
+  germline: "../../config/config_germline_filter.yaml"
+  cnv_hardfilter: "/../../config_cnv_hardfilter.yaml"
 
-germline_vcf:
-  container: "docker://marrip/vep:104.3"
-  filter: '--filter "DP > 20" --filter "MAX_AF >= 0.001"'
+fusioncatcher:
+  genome_path: "reference"
 
-imagemagick_convert:
-  container: "docker://hydragenetics/imagemagick:7.1.0_20"
+gatk_cnv_collect_allelic_counts:
+  SNP_interval: "reference/wgs.interval_list"
 
-manta_run_workflow_tn:
-  bed_files:
-    all: "reference/all.bed"
-    aml: "reference/aml.bed"
-  container: "docker://marrip/manta:v1.6.0"
+gatk_cnv_denoise_read_counts_by_sex:
+  pon_female: "reference/female.hdf5"
+  pon_male: "reference/male.hdf5"
 
-mosdepth:
-  by: "500"
-  container: "docker://hydragenetics/mosdepth:0.3.2"
-  extra: "--fast-mode --no-per-base"
+mosdepth_bed:
+  design_bed: "reference/exome.bed"
 
 multiqc:
-  container: "docker://marrip/multiqc:1.11"
   reports:
     DNA:
-      included_unit_types: ['T', 'N']
-      qc_files:
-        - "qc/fastqc/{sample}_{type}_{flowcell}_{lane}_{barcode}_{read}_fastqc.zip"
-        - "qc/mosdepth/{sample}_{type}.mosdepth.global.dist.txt"
-        - "qc/mosdepth/{sample}_{type}.mosdepth.region.dist.txt"
-        - "qc/mosdepth/{sample}_{type}.mosdepth.summary.txt"
-        - "qc/mosdepth/{sample}_{type}.regions.bed.gz"
-        - "qc/mosdepth/{sample}_{type}.regions.bed.gz.csi"
-        - "qc/picard_collect_alignment_summary_metrics/{sample}_{type}.alignment_summary_metrics.txt"
-        - "qc/picard_collect_duplication_metrics/{sample}_{type}.duplication_metrics.txt"
-        - "qc/picard_collect_gc_bias_metrics/{sample}_{type}.gc_bias.pdf"
-        - "qc/picard_collect_gc_bias_metrics/{sample}_{type}.gc_bias.detail_metrics"
-        - "qc/picard_collect_gc_bias_metrics/{sample}_{type}.gc_bias.summary_metrics"
-        - "qc/picard_collect_multiple_metrics/{sample}_{type}.alignment_summary_metrics"
-        - "qc/picard_collect_multiple_metrics/{sample}_{type}.base_distribution_by_cycle_metrics"
-        - "qc/picard_collect_multiple_metrics/{sample}_{type}.base_distribution_by_cycle.pdf"
-        - "qc/picard_collect_multiple_metrics/{sample}_{type}.insert_size_metrics"
-        - "qc/picard_collect_multiple_metrics/{sample}_{type}.insert_size_histogram.pdf"
-        - "qc/picard_collect_multiple_metrics/{sample}_{type}.quality_by_cycle_metrics"
-        - "qc/picard_collect_multiple_metrics/{sample}_{type}.quality_by_cycle.pdf"
-        - "qc/picard_collect_multiple_metrics/{sample}_{type}.quality_distribution_metrics"
-        - "qc/picard_collect_multiple_metrics/{sample}_{type}.quality_distribution.pdf"
-        - "qc/picard_collect_wgs_metrics/{sample}_{type}.txt"
-        - "qc/samtools_stats/{sample}_{type}.samtools-stats.txt"
-
-pbrun_mutectcaller_tn:
-  bed_files:
-    all: "reference/all.bed"
-    aml: "reference/aml.bed"
-
-picard_collect_alignment_summary_metrics:
-  container: "docker://hydragenetics/picard:2.25.0"
-
-picard_collect_duplication_metrics:
-  container: "docker://hydragenetics/picard:2.25.0"
-
-picard_collect_gc_bias_metrics:
-  container: "docker://hydragenetics/picard:2.25.0"
-
-picard_collect_multiple_metrics:
-  container: "docker://hydragenetics/picard:2.25.0"
-  output_ext:
-    - "alignment_summary_metrics"
-    - "base_distribution_by_cycle_metrics"
-    - "base_distribution_by_cycle.pdf"
-    - "insert_size_metrics"
-    - "insert_size_histogram.pdf"
-    - "quality_by_cycle_metrics"
-    - "quality_by_cycle.pdf"
-    - "quality_distribution_metrics"
-    - "quality_distribution.pdf"
-
-picard_collect_wgs_metrics:
-  container: "docker://hydragenetics/picard:2.25.0"
+      config: "../../config/multiqc_dna_config.yaml"
+    RNA:
+      config: "../../config/multiqc_rna_config.yaml"
+
+peddy:
+  config: "peddy_mqc.tsv" 
 
 pindel_call:
-  bedfile: "reference/pindel.bed"
-  container: "docker://hydragenetics/pindel:0.2.5b9"
+  include_bed: "reference/pindel.bed"
+
+rseqc_gene_body_coverage:
+  bed: "reference/gene_model.bed"
+
+rseqc_inner_distance:
+  bed: "reference/gene_model.bed"
 
-pindel2vcf:
-  container: "docker://hydragenetics/pindel:0.2.5b9"
+sample_order_multiqc:
+  sample_sheet: "SampleSheet.csv"
 
 simple_sv_annotation:
-  container: "docker://hydragenetics/simple_sv_annotation:2019.02.18"
   panel: "reference/panel.txt"
   fusion_pairs: "reference/fusion_pairs.txt"
 
 snpeff:
-  container: "docker://hydragenetics/snpeff:5.0"
   db: "SnpEff/GRCh38.99"
 
-spring:
-  container: "docker://hydragenetics/spring:1.0.1"
+sortmerna:
+  fasta:
+    - "reference/id.fasta"
+  index: "reference/"
+
+star:
+  genome_index: "reference/"
+
+star_fusion:
+  genome_path: "reference/"
+
+svdb_merge:
+  tc_method:
+    - name: pathology
+      cnv_caller:
+        - cnvkit
+        - gatk
 
 vep:
-  container: "docker://marrip/vep:104.3"
   vep_cache: "reference/vep"

.tests/integration/reference/hg38_CNA.bed

@@ -0,0 +1 @@
+