From 148f1fdbbfd7ab7fa1170d94f7c5064bff54a8ff Mon Sep 17 00:00:00 2001
From: Benjamin Wingfield <bwingfield@ebi.ac.uk>
Date: Wed, 10 Jul 2024 10:54:49 +0100
Subject: [PATCH 01/31] Remove references to multiple cohorts

---
 docs/how-to/samplesheet.rst | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/docs/how-to/samplesheet.rst b/docs/how-to/samplesheet.rst
index 49cb648f..ede58d8b 100644
--- a/docs/how-to/samplesheet.rst
+++ b/docs/how-to/samplesheet.rst
@@ -61,12 +61,11 @@ There are four mandatory columns:
 Notes
 ~~~~~
 
-.. note:: Multiple samplesheet rows are typically only needed if:
-          
-          - The target genomes are split to have a one file per chromosome
-          - You're working with multiple cohorts simultaneously 
+.. danger:: Always include every target genome chromosome in your samplesheet unless you're certain that missing chromosomes aren't in the scoring files 
+
+.. note:: Multiple samplesheet rows are typically only needed if the target genomes are split to have a one file per chromosome
 
-.. danger:: All samplesets have to be in the same genome build (either GRCh37 or
+.. danger:: All target genome files have to be in the same genome build (either GRCh37 or
     GRCh38) which is specified using the ``--target_build [GRCh3#]``
     command. All scoring files are downloaded or mapped to match the specified
     genome build, no liftover/re-mapping of the genotyping data is performed
@@ -90,10 +89,7 @@ There is one optional column:
   imputation tools (Michigan or TopMed Imputation Servers) that output dosages for the
   ALT allele(s): to extract these data users should enter ``DS`` in this column.
 
-An example of a samplesheet with two VCF datasets where you'd like to import
-different genotypes from each is below:
-
-.. list-table:: Example samplesheet with genotype field set
+.. list-table:: Example samplesheet with genotype field set to hard-calls (default)
    :header-rows: 1
 
    * - sampleset
@@ -106,6 +102,15 @@ different genotypes from each is below:
      - 22
      - vcf
      - ``GT``
+
+.. list-table:: Example samplesheet with genotype field set to dosage
+   :header-rows: 1
+
+   * - sampleset
+     - path_prefix
+     - chrom
+     - format 
+     - vcf_genotype_field       
    * - cineca_imputed
      - path/to/vcf_imputed
      - 22

From 501bf26dd77aec886cb184d7ed358045d7a915d7 Mon Sep 17 00:00:00 2001
From: Benjamin Wingfield <bwingfield@ebi.ac.uk>
Date: Mon, 15 Jul 2024 14:54:55 +0100
Subject: [PATCH 02/31] Update container tags to use ghcr

---
 conf/modules.config | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/conf/modules.config b/conf/modules.config
index 8c341266..ebb54516 100644
--- a/conf/modules.config
+++ b/conf/modules.config
@@ -52,18 +52,18 @@ process {
 
     withLabel: zstd {
         ext.conda = "$projectDir/environments/zstd/environment.yml"
-        ext.singularity = 'oras://dockerhub.ebi.ac.uk/gdp-public/pgsc_calc/singularity/zstd'
-        ext.singularity_version = ':amd64-1.4.8'
-        ext.docker = 'dockerhub.ebi.ac.uk/gdp-public/pgsc_calc/zstd'
-        ext.docker_version = ':1.4.8'
+        ext.singularity = 'oras://ghcr.io/pgscatalog/zstd'
+        ext.singularity_version = ':2-beta-singularity'
+        ext.docker = 'ghcr.io/pgscatalog/zstd'
+        ext.docker_version = ':2-beta'
     }
 
     withLabel: report {
         ext.conda = "$projectDir/environments/report/environment.yml"
-        ext.singularity = 'oras://dockerhub.ebi.ac.uk/gdp-public/pgsc_calc/singularity/report'
-        ext.singularity_version = ':2.00a5'
-        ext.docker = 'dockerhub.ebi.ac.uk/gdp-public/pgsc_calc/report'
-        ext.docker_version = ':2.00a5'
+        ext.singularity = 'oras://ghcr.io/pgscatalog/report'
+        ext.singularity_version = ':2-beta-singularity'
+        ext.docker = 'ghcr.io/pgscatalog/report'
+        ext.docker_version = ':2-beta'
     }
 
     withLabel: pyyaml {

From 38a835c12478623bdbc160726a0fa7818b634ad4 Mon Sep 17 00:00:00 2001
From: Benjamin Wingfield <bwingfield@ebi.ac.uk>
Date: Wed, 17 Jul 2024 11:11:48 +0100
Subject: [PATCH 03/31] read bim files with --stu_filt_iid

---
 modules/local/ancestry/oadp/fraposa_project.nf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/local/ancestry/oadp/fraposa_project.nf b/modules/local/ancestry/oadp/fraposa_project.nf
index acc831f8..0154c400 100644
--- a/modules/local/ancestry/oadp/fraposa_project.nf
+++ b/modules/local/ancestry/oadp/fraposa_project.nf
@@ -33,7 +33,7 @@ process FRAPOSA_PROJECT {
         --method $params.projection_method \
         --dim_ref 10 \
         --stu_filepref ${target_geno.baseName} \
-        --stu_filt_iid <(cut -f1 $split_fam) \
+        --stu_filt_iid $split_fam \
         --out ${target_geno.baseName}_${split_fam}
 
     cat <<-END_VERSIONS > versions.yml

From 093727d8523d56a066ba3766afb631db5ad34902 Mon Sep 17 00:00:00 2001
From: Benjamin Wingfield <bwingfield@ebi.ac.uk>
Date: Wed, 17 Jul 2024 11:12:15 +0100
Subject: [PATCH 04/31] write fid in plink2 --score

---
 modules/local/plink2_score.nf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/local/plink2_score.nf b/modules/local/plink2_score.nf
index 8842bede..6e250f2e 100644
--- a/modules/local/plink2_score.nf
+++ b/modules/local/plink2_score.nf
@@ -45,7 +45,7 @@ process PLINK2_SCORE {
     def no_imputation = ((ref_afreq.name == 'NO_FILE') && (meta.n_samples.toInteger() < 50)) ? "no-mean-imputation" : ""
     def error_on_freq_calc = (no_imputation == "no-mean-imputation") ? "--error-on-freq-calc" : ""
 
-    def cols = 'header-read cols=+scoresums,+denom,-fid'
+    def cols = 'header-read cols=+scoresums,+denom,+fid'
     def recessive = (scoremeta.effect_type == 'recessive') ? ' recessive ' : ''
     def dominant = (scoremeta.effect_type == 'dominant') ? ' dominant ' : ''
     args2 = [args2, cols, 'list-variants', no_imputation, recessive, dominant, error_on_freq_calc].join(' ')

From 0763fcdad827b81846a89918da105485bbfad952 Mon Sep 17 00:00:00 2001
From: Benjamin Wingfield <bwingfield@ebi.ac.uk>
Date: Fri, 19 Jul 2024 14:59:01 +0100
Subject: [PATCH 05/31] split the fam output, not the psam

---
 modules/local/ancestry/oadp/plink2_makebed.nf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/local/ancestry/oadp/plink2_makebed.nf b/modules/local/ancestry/oadp/plink2_makebed.nf
index 118b7716..cac77a46 100644
--- a/modules/local/ancestry/oadp/plink2_makebed.nf
+++ b/modules/local/ancestry/oadp/plink2_makebed.nf
@@ -48,11 +48,11 @@ process PLINK2_MAKEBED {
         --pvar $variants \
         --make-bed \
         $extract \
-        --out ${output}
+        --out ${output} \
 
     if [ $meta.id != 'reference' ]
     then
-        split -l 50000 <(grep -v '#' $pheno) ${split_output}
+        split -l 50000 ${output}.fam ${split_output}
     fi
 
     cat <<-END_VERSIONS > versions.yml

From 8dc174e47c76562b5926d65d8b4ccbe1c1f34bcb Mon Sep 17 00:00:00 2001
From: Benjamin Wingfield <bwingfield@ebi.ac.uk>
Date: Fri, 19 Jul 2024 16:07:34 +0100
Subject: [PATCH 06/31] fix report support for FID

---
 assets/report/report.qmd | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/assets/report/report.qmd b/assets/report/report.qmd
index b608ae49..5c7a69ac 100644
--- a/assets/report/report.qmd
+++ b/assets/report/report.qmd
@@ -443,7 +443,7 @@ uscores_plot <- uscores[1:min(length(uscores), 6)] # plot max 6 PGS
 
 # Plot multiple adjustment methods at once per PGS
 for(current_pgs in uscores_plot){
-  long_scores <- scores %>% select(!percentile_MostSimilarPop) %>% filter(PGS == current_pgs) %>% gather(Method, score, -sampleset, -IID, -PGS)
+  long_scores <- scores %>% select(!percentile_MostSimilarPop) %>% filter(PGS == current_pgs) %>% gather(Method, score, -sampleset, -FID, -IID, -PGS)
   long_scores %>%
   ggplot(aes(x = score, fill = sampleset)) +
     geom_density(alpha = 0.3) +

From 493c80f57995a73b3692dc760097185aeae6b316 Mon Sep 17 00:00:00 2001
From: Benjamin Wingfield <bwingfield@ebi.ac.uk>
Date: Fri, 19 Jul 2024 16:10:01 +0100
Subject: [PATCH 07/31] fix line ending

---
 modules/local/ancestry/oadp/plink2_makebed.nf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/local/ancestry/oadp/plink2_makebed.nf b/modules/local/ancestry/oadp/plink2_makebed.nf
index cac77a46..67f8a3f1 100644
--- a/modules/local/ancestry/oadp/plink2_makebed.nf
+++ b/modules/local/ancestry/oadp/plink2_makebed.nf
@@ -48,7 +48,7 @@ process PLINK2_MAKEBED {
         --pvar $variants \
         --make-bed \
         $extract \
-        --out ${output} \
+        --out ${output}
 
     if [ $meta.id != 'reference' ]
     then

From 6fbb82946d5201e0fbf4943bbfff49ac2414e3bb Mon Sep 17 00:00:00 2001
From: Benjamin Wingfield <bwingfield@ebi.ac.uk>
Date: Mon, 22 Jul 2024 17:14:07 +0100
Subject: [PATCH 08/31] bump fraposa version

---
 conf/modules.config                  | 4 ++--
 environments/fraposa/environment.yml | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/conf/modules.config b/conf/modules.config
index ebb54516..81ebf32f 100644
--- a/conf/modules.config
+++ b/conf/modules.config
@@ -77,9 +77,9 @@ process {
     withLabel: fraposa {
         ext.conda = "$projectDir/environments/fraposa/environment.yml"
         ext.singularity = 'oras://ghcr.io/pgscatalog/fraposa_pgsc'
-        ext.singularity_version = ':v0.1.0-singularity'
+        ext.singularity_version = ':v1.0.0-singularity'
         ext.docker = 'ghcr.io/pgscatalog/fraposa_pgsc'
-        ext.docker_version = ':v0.1.1'
+        ext.docker_version = ':v1.0.0'
     }
 
     // output configuration
diff --git a/environments/fraposa/environment.yml b/environments/fraposa/environment.yml
index b5c27de0..34612e75 100644
--- a/environments/fraposa/environment.yml
+++ b/environments/fraposa/environment.yml
@@ -3,4 +3,4 @@ channels:
   - conda-forge
   - bioconda
 dependencies:
-- fraposa-pgsc=0.1.1
\ No newline at end of file
+- fraposa-pgsc=1.0.0
\ No newline at end of file

From 913b191e70248eef4d44f1c9121a4ec6fbaf2f98 Mon Sep 17 00:00:00 2001
From: Benjamin Wingfield <bwingfield@ebi.ac.uk>
Date: Wed, 24 Jul 2024 11:07:20 +0100
Subject: [PATCH 09/31] remove duplicated changelog

---
 docs/_templates/globaltoc.html |   2 +-
 docs/changelog.rst             | 332 ---------------------------------
 2 files changed, 1 insertion(+), 333 deletions(-)
 delete mode 100644 docs/changelog.rst

diff --git a/docs/_templates/globaltoc.html b/docs/_templates/globaltoc.html
index f54562ec..048f9eef 100644
--- a/docs/_templates/globaltoc.html
+++ b/docs/_templates/globaltoc.html
@@ -22,7 +22,7 @@ <h3>Contents</h3>
 <h3>About the project</h3>
 
 <ul>
-  <li><a href="{{ pathto('changelog') }}">Changelog</a></li>
+  <li><a href="https://github.com/PGScatalog/pgsc_calc/releases">Changelog</a></li>
   <li><a href="{{ pathto('license') }}">License</a></li>
 </ul>
 
diff --git a/docs/changelog.rst b/docs/changelog.rst
deleted file mode 100644
index 676c6a06..00000000
--- a/docs/changelog.rst
+++ /dev/null
@@ -1,332 +0,0 @@
-:orphan:
-
-Changelog
----------
-
-Versions follow `semantic versioning`_ (``major.minor.patch``). Breaking changes
-will only occur in major versions with changes noted in this changelog.
-
-.. _`semantic versioning`: https://semver.org/
-
-pgsc_calc v2.0.0-beta (2024-06-19)
--------------------------------------
-
-Graduating to beta with the release of `our preprint <https://www.medrxiv.org/content/10.1101/2024.05.29.24307783v1>`_ 🎉
-
-Improvements
-
-* https://github.com/PGScatalog/pygscatalog/pull/23 
-
-* https://github.com/PGScatalog/pygscatalog/pull/22
-
-* https://github.com/PGScatalog/pgsc_calc/pull/311
-
-* Publish dependencies to Bioconda to improve conda profile UX
-
-  * https://anaconda.org/bioconda/fraposa-pgsc
-
-  * https://anaconda.org/bioconda/pgscatalog.core
-
-  * https://anaconda.org/bioconda/pgscatalog.match
-
-  * https://anaconda.org/bioconda/pgscatalog.calc
-
-Bug fixes 
-
-* Fix for https://github.com/PGScatalog/pygscatalog/issues/21
-
-* Closes  https://github.com/PGScatalog/pgsc_calc/pull/301
-
-* Specify modules explicitly to fix  https://github.com/PGScatalog/pgsc_calc/pull/312  
-
-* Fix bim input to `pgscatalog-aggregate`  https://github.com/PGScatalog/pgsc_calc/pull/319
-
-
-pgsc_calc v2.0.0-alpha.6 (2024-05-24)
--------------------------------------
-
-Please note the minimum required nextflow version has been updated to v23.10.0, released in October 2023.
-
-Improvements
-
-* Migrate our custom python tools to new https://github.com/PGScatalog/pygscatalog packages
-
-  * Reference / target intersection now considers allelic frequency and variant missingness to determine PCA eligibility
-
-  * Downloads from PGS Catalog should be faster (async)
-
-  * Package CLI and libraries `are now documented <https://pygscatalog.readthedocs.io/en/latest/?badge=latest>`_
-
-* Update plink version to alpha 5.10 final 
-
-* Add docs describing cloud execution 
-
-* Add correlation test comparing calculated scores against known good scores
-
-* When matching variants, matching logs are now written before scorefiles to improve debugging UX
-
-* Improvements to PCA quality (ensuring low missingness and suitable MAF for PCA-eligble variants in target samples).
-
-  *  This could allow us to implement MAF/missingness filters for scoring file variants in the future. 
-
-Bug fixes
-
-* Fix ancestry adjustment with VCFs
-* Fix support for scoring files that only have one effect type column 
-* Fix adjusting PGS with zero variance (skip them) 
-* Check for reserved characters in sampleset names
-
-pgsc_calc v2.0.0-alpha.5 (2024-03-19)
--------------------------------------
-
-Improvements:
-
-* Automatically mount directories inside singularity containers without setting any configuration
-* Improve permanent caching of ancestry processes with --genotypes_cache parameter
-* resync with nf-core framework
-* Refactor combine_scorefiles
-
-Bug fixes:
-
-* Fix semantic storeDir definitions causing problems cloud execution (google batch)
-* Fix missing DENOM values with multiple custom scoring files (score calculation not affected)
-* Fix liftover failing silently with custom scoring files (thanks Brooke!)
-
-Misc:
-
-* Move aggregation step out of report
-
-pgsc_calc v2.0.0-alpha.4 (2023-12-05)
--------------------------------------
-
-Improvements:
-
-* Give a more helpful error message when there's no valid variant matches found
-
-Bug fixes:
-
-* Fix retrying downloads from PGS Catalog
-* Fix numeric sample identifiers breaking ancestry analysis
-* Check for chr prefix in samplesheets and error 
-
-pgsc_calc v2.0.0-alpha.3 (2023-10-02)
--------------------------------------
-
-Improvements:
-
-* Automatically retry scoring with more RAM on larger datasets
-* Describe scoring precision in docs 
-* Change handling of VCFs to reduce errors when recoding 
-* Internal changes to improve support for custom reference panels
-
-Bug fixes:
-
-* Fix VCF input to ancestry projection subworkflow (thanks `@frahimov`_ and `@AWS-crafter`_ for patiently debugging)
-* Fix scoring options when reading allelic frequencies from a reference panel (thanks `@raimondsre`_ for reporting the changes from v1.3.2 -> 2.0.0-alpha)
-* Fix conda profile action
-
-.. _`@frahimov`: https://github.com/PGScatalog/pgsc_calc/issues/172
-.. _`@AWS-crafter`: https://github.com/PGScatalog/pgsc_calc/issues/155
-.. _`@raimondsre`: https://github.com/PGScatalog/pgsc_calc/pull/139#issuecomment-1736313211
-
-pgsc_calc v2.0.0-alpha.1 (2023-08-11)
--------------------------------------
-
-This patch fixes a bug when running the workflow directly from github with the
-test profile (i.e. without cloning first). Thanks to `@staedlern`_ for reporting the
-problem.
-
-.. _`@staedlern`: https://github.com/PGScatalog/pgsc_calc/issues/151
-
-pgsc_calc v2.0.0-alpha (2023-08-08)
------------------------------------
-
-This major release features breaking changes to samplesheet structure to provide
-more flexible support for extra genomic file types in the future. Two major new
-features were implemented in this release:
-
-- Genetic ancestry group similarity is calculated to a population reference panel
-  (default: 1000 Genomes) when the ``--run_ancestry`` flag is supplied. This runs
-  using PCA and projection implemented in the ``fraposa_pgsc (v0.1.0)`` package.
-- Calculated PGS can be adjusted for genetic ancestry using empirical PGS distributions
-  from the most similar reference panel population or continuous PCA-based regressions.
-
-These new features are optional and don't run in the default workflow. Other features
-included in the release are:
-
-- Speed optimizations for PGS scoring (skipping allele frequency calculation)
-
-pgsc_calc v1.3.2 (2023-01-27)
------------------------------
-
-This patch fixes a bug that made some PGS Catalog scoring files incompatible
-with the pipeline. Effect weights were sometimes set to utf-8 strings instead of
-floating point numbers, which caused an assertion error. Thanks to `@j0n-a`_ for
-reporting the problem.
-
-.. _`@j0n-a`: https://github.com/PGScatalog/pgsc_calc/issues/79
-
-pgsc_calc v1.3.1 (2023-01-24)
------------------------------
-
-This patch fixes a bug that breaks the workflow if all variants in one or more
-PGS scoring files match perfectly with the target genomes. Thanks to
-`@lemieuxl`_ for reporting the problem!
-
-.. _`@lemieuxl`: https://github.com/PGScatalog/pgsc_calc/issues/75
-
-pgsc_calc v1.3.0 (2022-11-21)
------------------------------
-
-This release is focused on improving scalability.
-
-Features
-~~~~~~~~
-
-- Variant matching is made more efficient using a split - apply - combine
-  approach when the data is split across chromosomes. This supports parallel PGS
-  calculation for the largest traits (e.g. cancer, 418 PGS [avg 261,000
-  variants/score) ) in the PGS Catalog on big datasets such as UK Biobank.
-
-- Better support for running in offline environments:
-
-  - Internet access is only required to download scores by ID. Scores can be
-    pre-downloaded using the utils package
-    (https://pypi.org/project/pgscatalog-utils/)
-
-  - Scoring file metadata is read from headers and displayed in the report
-    (removed API calls during report generation)
-
-- Implemented flag (--efo_direct) to return only PGS tagged with exact EFO term
-  (e.g. no PGS for child/descendant terms in the ontology)
-
-pgsc_calc v1.2.0 (2022-10-11)
------------------------------
-
-This release is focused on improving memory and storage usage.
-
-Features
-~~~~~~~~
-
-- Allow genotype dosages to be imported from VCF to be specified in ``vcf_genotype_field``
-  of samplesheet_ (default: GT / hard calls)
-
-- Makes use of `durable caching`_ when relabelling and recoding target genomes (``--genotypes_cache``)
-
-- Improvements to use less storage space:
-
-  - All intermediate files are now compressed by default
-
-  - Add parameter to support zstd compressed input files
-
-- Improved memory usage when matching variants (``pgscatalog_utils=v0.1.2``
-  https://github.com/PGScatalog/pgscatalog_utils)
-
-- Revised interface to select scores from the PGS Catalog using flags:
-  ``--trait_efo`` (EFO ID / traits), ``--pgp_id`` (PGP ID / publications), ``--pgs_id`` (PGS ID, individual scores).
-
-.. _samplesheet: https://pgsc-calc.readthedocs.io/en/dev/reference/input.html
-.. _durable caching: https://pgsc-calc.readthedocs.io/en/dev/reference/params.html#parameter-schema
-
-pgsc_calc v1.1.0 (2022-09-15)
------------------------------
-
-The first public release of the pgsc_calc pipeline. This release adds compatibility
-for every score published in the PGS Catalog. Each scoring file in the PGS Catalog
-has been processed to provide consistent genomic coordinates in builds GRCh37 and GRCh38.
-The pipeline has been updated to take advantage of the harmonised scoring files (see
-`PGS Catalog downloads`_ for additional details).
-
-.. _PGS Catalog downloads: https://www.pgscatalog.org/downloads/#dl_ftp_scoring_hm_pos
-
-Features
-~~~~~~~~
-
-- Many of the underlying software tools are now implemented within a ``pgscatalog_utils``
-  package (``v0.1.2``, https://github.com/PGScatalog/pgscatalog_utils and
-  https://pypi.org/project/pgscatalog-utils/ ). The packaging allows for independent
-  testing and development of tools for downloading and working with the scoring files.
-
-- The output report has been improved to have more detailed metadata describing
-  the scoring files and how well the variants match the target sampleset(s).
-
-- Improvements to variant matching:
-    - More precise control of variant matching parameters is now possible, like
-      ignoring strand flips
-    - ``match_variants`` should now use less RAM by default:
-        - A laptop with 16GB of RAM should be able to comfortably calculate scores on
-          the 1000 genomes dataset
-        - Fast matching mode (``--fast_match``) is available if ~32GB of RAM is
-          available and you'd like to calculate scores for larger datasets
-
-- Groups of scores from the PGS Catalog can be calculated by specifying a specific
-  ``--trait`` (EFO ID) or ``--publication`` (PGP ID), in addition to using individual
-  scoring files ``--pgs_id`` (PGS ID).
-
-- Score validation has been integrated with the test suite
-
-- Support for M1 Macs with ``--platform`` parameter (docker executor only)
-
-
-Bug fixes
-~~~~~~~~~
-
-- Implemented a more robust prioritisation procedure if a variant has multiple
-  candidate matches or duplicated IDs
-
-- Fixed processing multiple samplesets in parallel (e.g. 1000 Genomes + UK
-  Biobank)
-
-- When combining multiple scoring files, all variants are now kept to reflect the
-  correct denominator for % matching statistics.
-
-- When trying to correct for strand flips the matched effect allele wasn't being
-  correctly complemented
-
-pgsc_calc v1.0.0 (2022-05-24)
---------------------------------
-
-This release produces scores that should be biologically meaningful. Significant
-effort has been made to validate calculate scores on different datasets. In the
-next release we'll add score validation to our test suite to make sure
-calculated scores stay valid in the future.
-
-Features
-~~~~~~~~
-
-- Add support for PLINK2 format (samplesheet structure changed)
-- Add support for allosomes (e.g. X, Y)
-- Improve PGS Catalog compatibility (e.g. missing other allele)
-- Add automatic liftover of scoring files to match target genome build
-- Performance improvements to support UK BioBank scale data (500,000 genomes)
-- Support calculation of multiple scores in parallel
-- Significantly improved test coverage (> 80%)
-- Lots of other small changes to improve correctness and handling edge cases
-
-pgsc_calc v0.1.3dev (2022-02-04)
---------------------------------
-
-Features
-~~~~~~~~
-
-- Simplified JSON input processes
-- Add first draft of documentation
-- Add JSON schemas for validating input data (mostly for web platform)
-
-pgsc_calc v0.1.2dev (2022-01-17)
---------------------------------
-
-Features
-~~~~~~~~
-
-- Add JSON input support for web platform functionality
-- Set up simple CI tests with Github actions
-
-pgsc_calc v0.1.1dev (2021-12-16)
---------------------------------
-
-Features
-~~~~~~~~
-
-- First public release
-- Support applying a single scoring file to target genomic data in GrCh37 build

From 860b12fa0ffc0c636fb2fecbc3a584288dc9f126 Mon Sep 17 00:00:00 2001
From: Benjamin Wingfield <bwingfield@ebi.ac.uk>
Date: Wed, 24 Jul 2024 11:07:20 +0100
Subject: [PATCH 10/31] remove duplicated changelog

---
 docs/_templates/globaltoc.html |   2 +-
 docs/changelog.rst             | 332 ---------------------------------
 2 files changed, 1 insertion(+), 333 deletions(-)
 delete mode 100644 docs/changelog.rst

diff --git a/docs/_templates/globaltoc.html b/docs/_templates/globaltoc.html
index f54562ec..048f9eef 100644
--- a/docs/_templates/globaltoc.html
+++ b/docs/_templates/globaltoc.html
@@ -22,7 +22,7 @@ <h3>Contents</h3>
 <h3>About the project</h3>
 
 <ul>
-  <li><a href="{{ pathto('changelog') }}">Changelog</a></li>
+  <li><a href="https://github.com/PGScatalog/pgsc_calc/releases">Changelog</a></li>
   <li><a href="{{ pathto('license') }}">License</a></li>
 </ul>
 
diff --git a/docs/changelog.rst b/docs/changelog.rst
deleted file mode 100644
index 676c6a06..00000000
--- a/docs/changelog.rst
+++ /dev/null
@@ -1,332 +0,0 @@
-:orphan:
-
-Changelog
----------
-
-Versions follow `semantic versioning`_ (``major.minor.patch``). Breaking changes
-will only occur in major versions with changes noted in this changelog.
-
-.. _`semantic versioning`: https://semver.org/
-
-pgsc_calc v2.0.0-beta (2024-06-19)
--------------------------------------
-
-Graduating to beta with the release of `our preprint <https://www.medrxiv.org/content/10.1101/2024.05.29.24307783v1>`_ 🎉
-
-Improvements
-
-* https://github.com/PGScatalog/pygscatalog/pull/23 
-
-* https://github.com/PGScatalog/pygscatalog/pull/22
-
-* https://github.com/PGScatalog/pgsc_calc/pull/311
-
-* Publish dependencies to Bioconda to improve conda profile UX
-
-  * https://anaconda.org/bioconda/fraposa-pgsc
-
-  * https://anaconda.org/bioconda/pgscatalog.core
-
-  * https://anaconda.org/bioconda/pgscatalog.match
-
-  * https://anaconda.org/bioconda/pgscatalog.calc
-
-Bug fixes 
-
-* Fix for https://github.com/PGScatalog/pygscatalog/issues/21
-
-* Closes  https://github.com/PGScatalog/pgsc_calc/pull/301
-
-* Specify modules explicitly to fix  https://github.com/PGScatalog/pgsc_calc/pull/312  
-
-* Fix bim input to `pgscatalog-aggregate`  https://github.com/PGScatalog/pgsc_calc/pull/319
-
-
-pgsc_calc v2.0.0-alpha.6 (2024-05-24)
--------------------------------------
-
-Please note the minimum required nextflow version has been updated to v23.10.0, released in October 2023.
-
-Improvements
-
-* Migrate our custom python tools to new https://github.com/PGScatalog/pygscatalog packages
-
-  * Reference / target intersection now considers allelic frequency and variant missingness to determine PCA eligibility
-
-  * Downloads from PGS Catalog should be faster (async)
-
-  * Package CLI and libraries `are now documented <https://pygscatalog.readthedocs.io/en/latest/?badge=latest>`_
-
-* Update plink version to alpha 5.10 final 
-
-* Add docs describing cloud execution 
-
-* Add correlation test comparing calculated scores against known good scores
-
-* When matching variants, matching logs are now written before scorefiles to improve debugging UX
-
-* Improvements to PCA quality (ensuring low missingness and suitable MAF for PCA-eligble variants in target samples).
-
-  *  This could allow us to implement MAF/missingness filters for scoring file variants in the future. 
-
-Bug fixes
-
-* Fix ancestry adjustment with VCFs
-* Fix support for scoring files that only have one effect type column 
-* Fix adjusting PGS with zero variance (skip them) 
-* Check for reserved characters in sampleset names
-
-pgsc_calc v2.0.0-alpha.5 (2024-03-19)
--------------------------------------
-
-Improvements:
-
-* Automatically mount directories inside singularity containers without setting any configuration
-* Improve permanent caching of ancestry processes with --genotypes_cache parameter
-* resync with nf-core framework
-* Refactor combine_scorefiles
-
-Bug fixes:
-
-* Fix semantic storeDir definitions causing problems cloud execution (google batch)
-* Fix missing DENOM values with multiple custom scoring files (score calculation not affected)
-* Fix liftover failing silently with custom scoring files (thanks Brooke!)
-
-Misc:
-
-* Move aggregation step out of report
-
-pgsc_calc v2.0.0-alpha.4 (2023-12-05)
--------------------------------------
-
-Improvements:
-
-* Give a more helpful error message when there's no valid variant matches found
-
-Bug fixes:
-
-* Fix retrying downloads from PGS Catalog
-* Fix numeric sample identifiers breaking ancestry analysis
-* Check for chr prefix in samplesheets and error 
-
-pgsc_calc v2.0.0-alpha.3 (2023-10-02)
--------------------------------------
-
-Improvements:
-
-* Automatically retry scoring with more RAM on larger datasets
-* Describe scoring precision in docs 
-* Change handling of VCFs to reduce errors when recoding 
-* Internal changes to improve support for custom reference panels
-
-Bug fixes:
-
-* Fix VCF input to ancestry projection subworkflow (thanks `@frahimov`_ and `@AWS-crafter`_ for patiently debugging)
-* Fix scoring options when reading allelic frequencies from a reference panel (thanks `@raimondsre`_ for reporting the changes from v1.3.2 -> 2.0.0-alpha)
-* Fix conda profile action
-
-.. _`@frahimov`: https://github.com/PGScatalog/pgsc_calc/issues/172
-.. _`@AWS-crafter`: https://github.com/PGScatalog/pgsc_calc/issues/155
-.. _`@raimondsre`: https://github.com/PGScatalog/pgsc_calc/pull/139#issuecomment-1736313211
-
-pgsc_calc v2.0.0-alpha.1 (2023-08-11)
--------------------------------------
-
-This patch fixes a bug when running the workflow directly from github with the
-test profile (i.e. without cloning first). Thanks to `@staedlern`_ for reporting the
-problem.
-
-.. _`@staedlern`: https://github.com/PGScatalog/pgsc_calc/issues/151
-
-pgsc_calc v2.0.0-alpha (2023-08-08)
------------------------------------
-
-This major release features breaking changes to samplesheet structure to provide
-more flexible support for extra genomic file types in the future. Two major new
-features were implemented in this release:
-
-- Genetic ancestry group similarity is calculated to a population reference panel
-  (default: 1000 Genomes) when the ``--run_ancestry`` flag is supplied. This runs
-  using PCA and projection implemented in the ``fraposa_pgsc (v0.1.0)`` package.
-- Calculated PGS can be adjusted for genetic ancestry using empirical PGS distributions
-  from the most similar reference panel population or continuous PCA-based regressions.
-
-These new features are optional and don't run in the default workflow. Other features
-included in the release are:
-
-- Speed optimizations for PGS scoring (skipping allele frequency calculation)
-
-pgsc_calc v1.3.2 (2023-01-27)
------------------------------
-
-This patch fixes a bug that made some PGS Catalog scoring files incompatible
-with the pipeline. Effect weights were sometimes set to utf-8 strings instead of
-floating point numbers, which caused an assertion error. Thanks to `@j0n-a`_ for
-reporting the problem.
-
-.. _`@j0n-a`: https://github.com/PGScatalog/pgsc_calc/issues/79
-
-pgsc_calc v1.3.1 (2023-01-24)
------------------------------
-
-This patch fixes a bug that breaks the workflow if all variants in one or more
-PGS scoring files match perfectly with the target genomes. Thanks to
-`@lemieuxl`_ for reporting the problem!
-
-.. _`@lemieuxl`: https://github.com/PGScatalog/pgsc_calc/issues/75
-
-pgsc_calc v1.3.0 (2022-11-21)
------------------------------
-
-This release is focused on improving scalability.
-
-Features
-~~~~~~~~
-
-- Variant matching is made more efficient using a split - apply - combine
-  approach when the data is split across chromosomes. This supports parallel PGS
-  calculation for the largest traits (e.g. cancer, 418 PGS [avg 261,000
-  variants/score) ) in the PGS Catalog on big datasets such as UK Biobank.
-
-- Better support for running in offline environments:
-
-  - Internet access is only required to download scores by ID. Scores can be
-    pre-downloaded using the utils package
-    (https://pypi.org/project/pgscatalog-utils/)
-
-  - Scoring file metadata is read from headers and displayed in the report
-    (removed API calls during report generation)
-
-- Implemented flag (--efo_direct) to return only PGS tagged with exact EFO term
-  (e.g. no PGS for child/descendant terms in the ontology)
-
-pgsc_calc v1.2.0 (2022-10-11)
------------------------------
-
-This release is focused on improving memory and storage usage.
-
-Features
-~~~~~~~~
-
-- Allow genotype dosages to be imported from VCF to be specified in ``vcf_genotype_field``
-  of samplesheet_ (default: GT / hard calls)
-
-- Makes use of `durable caching`_ when relabelling and recoding target genomes (``--genotypes_cache``)
-
-- Improvements to use less storage space:
-
-  - All intermediate files are now compressed by default
-
-  - Add parameter to support zstd compressed input files
-
-- Improved memory usage when matching variants (``pgscatalog_utils=v0.1.2``
-  https://github.com/PGScatalog/pgscatalog_utils)
-
-- Revised interface to select scores from the PGS Catalog using flags:
-  ``--trait_efo`` (EFO ID / traits), ``--pgp_id`` (PGP ID / publications), ``--pgs_id`` (PGS ID, individual scores).
-
-.. _samplesheet: https://pgsc-calc.readthedocs.io/en/dev/reference/input.html
-.. _durable caching: https://pgsc-calc.readthedocs.io/en/dev/reference/params.html#parameter-schema
-
-pgsc_calc v1.1.0 (2022-09-15)
------------------------------
-
-The first public release of the pgsc_calc pipeline. This release adds compatibility
-for every score published in the PGS Catalog. Each scoring file in the PGS Catalog
-has been processed to provide consistent genomic coordinates in builds GRCh37 and GRCh38.
-The pipeline has been updated to take advantage of the harmonised scoring files (see
-`PGS Catalog downloads`_ for additional details).
-
-.. _PGS Catalog downloads: https://www.pgscatalog.org/downloads/#dl_ftp_scoring_hm_pos
-
-Features
-~~~~~~~~
-
-- Many of the underlying software tools are now implemented within a ``pgscatalog_utils``
-  package (``v0.1.2``, https://github.com/PGScatalog/pgscatalog_utils and
-  https://pypi.org/project/pgscatalog-utils/ ). The packaging allows for independent
-  testing and development of tools for downloading and working with the scoring files.
-
-- The output report has been improved to have more detailed metadata describing
-  the scoring files and how well the variants match the target sampleset(s).
-
-- Improvements to variant matching:
-    - More precise control of variant matching parameters is now possible, like
-      ignoring strand flips
-    - ``match_variants`` should now use less RAM by default:
-        - A laptop with 16GB of RAM should be able to comfortably calculate scores on
-          the 1000 genomes dataset
-        - Fast matching mode (``--fast_match``) is available if ~32GB of RAM is
-          available and you'd like to calculate scores for larger datasets
-
-- Groups of scores from the PGS Catalog can be calculated by specifying a specific
-  ``--trait`` (EFO ID) or ``--publication`` (PGP ID), in addition to using individual
-  scoring files ``--pgs_id`` (PGS ID).
-
-- Score validation has been integrated with the test suite
-
-- Support for M1 Macs with ``--platform`` parameter (docker executor only)
-
-
-Bug fixes
-~~~~~~~~~
-
-- Implemented a more robust prioritisation procedure if a variant has multiple
-  candidate matches or duplicated IDs
-
-- Fixed processing multiple samplesets in parallel (e.g. 1000 Genomes + UK
-  Biobank)
-
-- When combining multiple scoring files, all variants are now kept to reflect the
-  correct denominator for % matching statistics.
-
-- When trying to correct for strand flips the matched effect allele wasn't being
-  correctly complemented
-
-pgsc_calc v1.0.0 (2022-05-24)
---------------------------------
-
-This release produces scores that should be biologically meaningful. Significant
-effort has been made to validate calculate scores on different datasets. In the
-next release we'll add score validation to our test suite to make sure
-calculated scores stay valid in the future.
-
-Features
-~~~~~~~~
-
-- Add support for PLINK2 format (samplesheet structure changed)
-- Add support for allosomes (e.g. X, Y)
-- Improve PGS Catalog compatibility (e.g. missing other allele)
-- Add automatic liftover of scoring files to match target genome build
-- Performance improvements to support UK BioBank scale data (500,000 genomes)
-- Support calculation of multiple scores in parallel
-- Significantly improved test coverage (> 80%)
-- Lots of other small changes to improve correctness and handling edge cases
-
-pgsc_calc v0.1.3dev (2022-02-04)
---------------------------------
-
-Features
-~~~~~~~~
-
-- Simplified JSON input processes
-- Add first draft of documentation
-- Add JSON schemas for validating input data (mostly for web platform)
-
-pgsc_calc v0.1.2dev (2022-01-17)
---------------------------------
-
-Features
-~~~~~~~~
-
-- Add JSON input support for web platform functionality
-- Set up simple CI tests with Github actions
-
-pgsc_calc v0.1.1dev (2021-12-16)
---------------------------------
-
-Features
-~~~~~~~~
-
-- First public release
-- Support applying a single scoring file to target genomic data in GrCh37 build

From 9f675c0e70bbbd11490e7e3f6f0c8b8a805f8410 Mon Sep 17 00:00:00 2001
From: Benjamin Wingfield <bwingfield@ebi.ac.uk>
Date: Mon, 29 Jul 2024 16:09:00 +0100
Subject: [PATCH 11/31] bump pgscatalog-utils version

---
 conf/modules.config                           | 4 ++--
 environments/pgscatalog_utils/environment.yml | 2 +-
 tests/modules/combine/test.yml                | 2 +-
 tests/modules/download/test.yml               | 4 ++--
 tests/modules/match/test.yml                  | 4 ++--
 tests/subworkflows/test_apply_score.py        | 2 +-
 tests/subworkflows/test_apply_score.yml       | 3 +++
 tests/subworkflows/test_liftover_run.yml      | 4 ++--
 8 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/conf/modules.config b/conf/modules.config
index 81ebf32f..1f345793 100644
--- a/conf/modules.config
+++ b/conf/modules.config
@@ -38,8 +38,8 @@ process {
         ext.conda = "$projectDir/environments/pgscatalog_utils/environment.yml"
         ext.docker = 'ghcr.io/pgscatalog/pygscatalog'
         ext.singularity = 'oras://ghcr.io/pgscatalog/pygscatalog'
-        ext.docker_version = ':pgscatalog-utils-1.1.2'
-        ext.singularity_version = ':pgscatalog-utils-1.1.2-singularity'
+        ext.docker_version = ':pgscatalog-utils-1.2.0'
+        ext.singularity_version = ':pgscatalog-utils-1.2.0-singularity'
     }
 
     withLabel: plink2 {
diff --git a/environments/pgscatalog_utils/environment.yml b/environments/pgscatalog_utils/environment.yml
index fac6ef15..fbf645ed 100644
--- a/environments/pgscatalog_utils/environment.yml
+++ b/environments/pgscatalog_utils/environment.yml
@@ -3,4 +3,4 @@ channels:
   - conda-forge
   - bioconda
 dependencies:
-- pgscatalog-utils=1.1.2
+- pgscatalog-utils=1.2.0
diff --git a/tests/modules/combine/test.yml b/tests/modules/combine/test.yml
index 0dcf4c16..ba6c16dc 100644
--- a/tests/modules/combine/test.yml
+++ b/tests/modules/combine/test.yml
@@ -14,4 +14,4 @@
         - "effect_weight"
     - path: output/combine/versions.yml
       contains:
-        - "pgscatalog.core: 0.2.1"
\ No newline at end of file
+        - "pgscatalog.core: 0.2.2"
\ No newline at end of file
diff --git a/tests/modules/download/test.yml b/tests/modules/download/test.yml
index 0d2bde30..54323166 100644
--- a/tests/modules/download/test.yml
+++ b/tests/modules/download/test.yml
@@ -8,7 +8,7 @@
     - path: output/download/PGS000001_hmPOS_GRCh37.txt.gz
     - path: output/download/versions.yml
       contains:
-        - "pgscatalog.core: 0.2.1"
+        - "pgscatalog.core: 0.2.2"
       
 - name: pgscatalog test --efo_trait --pgp_id and --pgs_id
   command: nextflow run ./tests/modules/download -entry testmultipleaccessions -c ./tests/config/nextflow.config
@@ -24,7 +24,7 @@
     - path: output/download/PGS002054_hmPOS_GRCh37.txt.gz
     - path: output/download/versions.yml
       contains:
-        - "pgscatalog.core: 0.2.1"
+        - "pgscatalog.core: 0.2.2"
 
 - name: pgscatalog test bad accession
   command: nextflow run ./tests/modules/download -entry testbadaccession -c ./tests/config/nextflow.config
diff --git a/tests/modules/match/test.yml b/tests/modules/match/test.yml
index 5db1804e..d0c41127 100644
--- a/tests/modules/match/test.yml
+++ b/tests/modules/match/test.yml
@@ -7,7 +7,7 @@
   files:
     - path: output/test/match/versions.yml
       contains:
-        - "pgscatalog.match: 0.2.2"
+        - "pgscatalog.match: 0.2.3"
 # can't test IPC output (not published)        
 
 - name: test match combine module
@@ -20,7 +20,7 @@
   files:
     - path: output/test/match/versions.yml
       contains:
-        - "pgscatalog.match: 0.2.2"
+        - "pgscatalog.match: 0.2.3"
     - path: output/test/match/test_ALL_additive_0.scorefile.gz           
       contains:
         - "PGS001229"     
diff --git a/tests/subworkflows/test_apply_score.py b/tests/subworkflows/test_apply_score.py
index 5575cf2f..6453caff 100644
--- a/tests/subworkflows/test_apply_score.py
+++ b/tests/subworkflows/test_apply_score.py
@@ -19,7 +19,7 @@ def test_aggregated_scores(workflow_dir):
 
     assert not df.isnull().any().any(), "Missing values in aggregated scores"
 
-    cols = ["sampleset", "IID", "PGS", "SUM", "DENOM", "AVG"]
+    cols = ["sampleset", "FID", "IID", "PGS", "SUM", "DENOM", "AVG"]
     assert cols == list(df.columns), "Missing columns"
     assert (
         len(
diff --git a/tests/subworkflows/test_apply_score.yml b/tests/subworkflows/test_apply_score.yml
index 637f37cf..9c322606 100644
--- a/tests/subworkflows/test_apply_score.yml
+++ b/tests/subworkflows/test_apply_score.yml
@@ -16,6 +16,9 @@
         - "IID"
         - "PGS"
         - "SUM"
+    - path: output/score/versions.yml
+      contains:
+        - "pgscatalog.calc: 0.2.2"
       must_not_contain:
         - "percentile_MostSimilarPop"
         - "Z_MostSimilarPop"
diff --git a/tests/subworkflows/test_liftover_run.yml b/tests/subworkflows/test_liftover_run.yml
index 1aa3f90a..e02c2010 100644
--- a/tests/subworkflows/test_liftover_run.yml
+++ b/tests/subworkflows/test_liftover_run.yml
@@ -11,7 +11,7 @@
       - "5297845"
     - path: output/combine/versions.yml      
       contains:
-        - "pgscatalog.core: 0.2.1"
+        - "pgscatalog.core: 0.2.2"
 
 - name: test input check subworkflow with liftover 37to38
   command: nextflow run main.nf --only_input --scorefile ./assets/examples/scorefiles/customgrch37.txt --liftover --target_build GRCh38 -c ./tests/config/nextflow.config --hg19_chain https://hgdownload.cse.ucsc.edu/goldenpath/hg19/liftOver/hg19ToHg38.over.chain.gz --hg38_chain https://hgdownload.soe.ucsc.edu/goldenPath/hg38/liftOver/hg38ToHg19.over.chain.gz
@@ -26,4 +26,4 @@
       - "5237785"
     - path: output/combine/versions.yml
       contains:
-        - "pgscatalog.core: 0.2.1"
+        - "pgscatalog.core: 0.2.2"

From 5b5692c1ad9a5ddc08c0d37e097452b26b7b4f10 Mon Sep 17 00:00:00 2001
From: Benjamin Wingfield <bwingfield@ebi.ac.uk>
Date: Mon, 29 Jul 2024 17:05:17 +0100
Subject: [PATCH 12/31] update schema with support for --maf_target and
 --geno_miss_target

---
 modules/local/ancestry/intersect_variants.nf |   4 +-
 nextflow.config                              |   2 +
 nextflow_schema.json                         | 355 +++++++++++--------
 3 files changed, 208 insertions(+), 153 deletions(-)

diff --git a/modules/local/ancestry/intersect_variants.nf b/modules/local/ancestry/intersect_variants.nf
index efcad0b6..d872bbf4 100644
--- a/modules/local/ancestry/intersect_variants.nf
+++ b/modules/local/ancestry/intersect_variants.nf
@@ -33,8 +33,8 @@ process INTERSECT_VARIANTS {
     pgscatalog-intersect --ref $ref_variants \
         --target $variants \
         --chrom $meta.chrom \
-        --maf_target 0.1 \
-        --geno_miss 0.1 \
+        --maf_target $params.maf_target \
+        --geno_miss $params.geno_miss_target \
         --outdir . \
         -v
 
diff --git a/nextflow.config b/nextflow.config
index 480ab095..27035ba5 100644
--- a/nextflow.config
+++ b/nextflow.config
@@ -43,6 +43,8 @@ params {
     n_popcomp = 5
     normalization_method = "empirical mean mean+var"
     n_normalization = 4
+    maf_target = 0.1
+    geno_miss_target = 0.1
 
     // compatibility params
     liftover = false
diff --git a/nextflow_schema.json b/nextflow_schema.json
index 6caed387..aefa1d31 100644
--- a/nextflow_schema.json
+++ b/nextflow_schema.json
@@ -1,33 +1,49 @@
 {
-  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$schema": "http://json-schema.org/draft-07/schema",
   "$id": "https://raw.githubusercontent.com/pgscatalog/pgsc_calc/master/nextflow_schema.json",
   "title": "pgscatalog/pgsc_calc pipeline parameters",
-  "description": "This pipeline applies scoring files from the PGS Catalog to target set(s) of genotyped samples",
+  "description": "The Polygenic Score Catalog Calculator is a nextflow pipeline for polygenic score calculation",
   "type": "object",
-  "defs": {
+  "definitions": {
     "input_output_options": {
       "title": "Input/output options",
       "type": "object",
       "fa_icon": "fas fa-terminal",
       "description": "Define where the pipeline should find input data and save output data.",
+      "required": ["input", "outdir"],
       "properties": {
         "input": {
           "type": "string",
-          "description": "Path to input samplesheet",
-          "format": "file-path"
+          "format": "file-path",
+          "exists": true,
+          "pattern": "^\\S+\\.(csv|json)$",
+          "description": "Path to comma-separated file containing information about the samples in the experiment.",
+          "help_text": "You will need to create a design file with information about the samples in your experiment before running the pipeline. Use this parameter to specify its location. It has to be a comma-separated file with 3 columns, and a header row.",
+          "fa_icon": "fas fa-file-csv"
+        },
+        "outdir": {
+          "type": "string",
+          "format": "directory-path",
+          "description": "The output directory where the results will be saved. You have to use absolute paths to storage on Cloud infrastructure.",
+          "fa_icon": "fas fa-folder-open",
+          "default": "results"
+        },
+        "email": {
+          "type": "string",
+          "description": "Email address for completion summary.",
+          "fa_icon": "fas fa-envelope",
+          "help_text": "Set this parameter to your e-mail address to get a summary e-mail with details of the run sent to you when the workflow exits. If set in your user config file (`~/.nextflow/config`) then you don't need to specify this on the command line for every run.",
+          "pattern": "^([a-zA-Z0-9_\\-\\.]+)@([a-zA-Z0-9_\\-\\.]+)\\.([a-zA-Z]{2,5})$"
         },
         "format": {
           "type": "string",
           "default": "csv",
-          "fa_icon": "fas fa-cog",
-          "description": "Format of input samplesheet",
-          "enum": ["csv", "json"]
+          "enum": ["csv", "json"],
+          "description": "What format is the samplesheet in? (csv/json)"
         },
         "scorefile": {
           "type": "string",
-          "description": "Path to a scoring file in PGS Catalog format. Multiple scorefiles can be specified using wildcards (e.g., ``--scorefile \"path/to/scores/*.txt\"``)",
-          "fa_icon": "fas fa-file-alt",
-          "format": "file-path"
+          "description": "Path to a scoring file in PGS Catalog format. Multiple scorefiles can be specified using wildcards (e.g., ``--scorefile \\\"path/to/scores/*.txt\\\"``)"
         },
         "pgs_id": {
           "type": "string",
@@ -55,35 +71,68 @@
           "type": "string",
           "description": "Path to a directory that can store relabelled genotypes (and the reference panel intersections and PCA with --run_ancestry) to speed up new PGS calculations on previously harmonized samples",
           "format": "directory-path"
-        },
-        "outdir": {
+        }
+      }
+    },
+    "compatibility_options": {
+      "title": "Compatibility options",
+      "type": "object",
+      "description": "Define parameters that control how scoring files and target genomes are made compatible with each other",
+      "default": "",
+      "properties": {
+        "target_build": {
           "type": "string",
-          "description": "Path to the output directory where the results will be saved.",
-          "fa_icon": "fas fa-folder-open",
-          "format": "directory-path",
-          "default": "results"
+          "description": "Genome build of target genomes",
+          "enum": ["GRCh37", "GRCh38"]
         },
-        "email": {
-          "type": "string",
-          "description": "Email address for completion summary.",
-          "fa_icon": "fas fa-envelope",
-          "help_text": "Set this parameter to your e-mail address to get a summary e-mail with details of the run sent to you when the workflow exits. If set in your user config file (`~/.nextflow/config`) then you don't need to specify this on the command line for every run.",
-          "pattern": "^([a-zA-Z0-9_\\-\\.]+)@([a-zA-Z0-9_\\-\\.]+)\\.([a-zA-Z]{2,5})$"
+        "liftover": {
+          "type": "boolean",
+          "description": "Lift scoring files to match your target genomes. Requires build information in the header of the scoring files."
+        },
+        "min_lift": {
+          "type": "number",
+          "default": 0.95,
+          "description": "Minimum proportion of variants required to successfully remap a scoring file to a different genome build.",
+          "minimum": 0,
+          "maximum": 1
         }
       },
-      "required": ["input", "format"]
+      "required": ["target_build"]
     },
-    "ancestry_options": {
-      "title": "Ancestry options",
+    "new_group_4": {
+      "title": "New Group 4",
       "type": "object",
-      "description": "",
+      "description": "Define how variants are matched across scoring files and target genomes.",
+      "default": "",
+      "properties": {
+        "keep_multiallelic": {
+          "type": "boolean",
+          "description": "Allow matches of scoring file variants to multiallelic variants in the target dataset"
+        },
+        "keep_ambiguous": {
+          "type": "boolean",
+          "description": "Keep matches of scoring file variants to strand ambiguous variants (e.g. A/T and C/G SNPs) in the target dataset. This assumes the scoring file and target dataset report variants on the same strand."
+        },
+        "min_overlap": {
+          "type": "number",
+          "default": 0.75,
+          "minimum": 0,
+          "maximum": 1,
+          "description": "Minimum proportion of variants present in both the score file and input target genomic data"
+        }
+      }
+    },
+    "genetic_ancestry_options": {
+      "title": "Genetic ancestry options",
+      "type": "object",
+      "description": "Parameters used to control genetic ancestry similarity analysis",
       "default": "",
       "properties": {
         "projection_method": {
           "type": "string",
           "default": "oadp",
-          "enum": ["oadp", "sp", "adp"],
-          "description": "The method for PCA prediction. oadp: most accurate. adp: accurate but slow. sp: fast but inaccurate."
+          "description": "The method for PCA prediction. oadp: most accurate. adp: accurate but slow. sp: fast but inaccurate.",
+          "enum": ["oadp", "sp", "adp"]
         },
         "ancestry_method": {
           "type": "string",
@@ -104,8 +153,8 @@
         "normalization_method": {
           "type": "string",
           "default": "empirical mean mean+var",
-          "description": "Method used for normalisation of genetic ancestry",
-          "enum": ["empirical", "mean", "mean+var", "empirical mean mean+var"]
+          "enum": ["empirical", "mean", "mean+var", "empirical mean mean+var"],
+          "description": "Method used for normalisation of genetic ancestry"
         },
         "n_normalization": {
           "type": "integer",
@@ -116,6 +165,14 @@
           "type": "boolean",
           "default": true,
           "description": "Load allelic frequencies from reference panel when scoring target genomes"
+        },
+        "pca_maf_target": {
+          "type": "number",
+          "default": 0.1
+        },
+        "pca_geno_miss": {
+          "type": "number",
+          "default": 0.1
         }
       },
       "required": [
@@ -125,7 +182,9 @@
         "n_popcomp",
         "normalization_method",
         "n_normalization",
-        "load_afreq"
+        "load_afreq",
+        "pca_maf_target",
+        "pca_geno_miss"
       ]
     },
     "reference_options": {
@@ -136,12 +195,13 @@
       "properties": {
         "run_ancestry": {
           "type": "string",
-          "format": "file-path",
-          "description": "Path to reference database. Must be set if --ref_samplesheet is not set."
+          "description": "Path to reference database. Must be set if --ref_samplesheet is not set.",
+          "format": "file-path"
         },
         "ref_samplesheet": {
           "type": "string",
-          "description": "Path to a samplesheet that describes the structure of reference data. Must be set if --ref isn't set."
+          "description": "Path to a samplesheet that describes the structure of reference data. Must be set if --ref isn't set.",
+          "format": "file-path"
         },
         "hg19_chain": {
           "type": "string",
@@ -153,6 +213,7 @@
         "hg38_chain": {
           "type": "string",
           "description": "Path to a UCSC chain file for converting from hg38 to hg19.  Needed if lifting over a custom scoring file.",
+          "pattern": ".*chain.gz$",
           "format": "file-path",
           "mimetype": "application/gzip"
         },
@@ -166,16 +227,16 @@
         "mind_ref": {
           "type": "number",
           "default": 0.1,
-          "description": "Exclude samples with missing call frequencies greater than a threshold (in reference genomes)",
           "minimum": 0,
-          "maximum": 1
+          "maximum": 1,
+          "description": "Exclude samples with missing call frequencies greater than a threshold (in reference genomes)"
         },
         "maf_ref": {
           "type": "number",
           "default": 0.05,
-          "description": "Exclude variants with allele frequency lower than a threshold (in reference genomes)",
           "minimum": 0,
-          "maximum": 1
+          "maximum": 1,
+          "description": "Exclude variants with allele frequency lower than a threshold (in reference genomes)"
         },
         "hwe_ref": {
           "type": "number",
@@ -191,83 +252,113 @@
         },
         "ld_grch37": {
           "type": "string",
+          "default": "/Users/bwingfield/Documents/projects/pgsc_calc/assets/ancestry/high-LD-regions-hg19-GRCh37.txt",
           "description": "Path to a file that contains areas of high linkage disequilibrium in the reference data (build GRCh37).",
           "format": "file-path",
           "mimetype": "text/plain"
         },
         "ld_grch38": {
           "type": "string",
+          "default": "/Users/bwingfield/Documents/projects/pgsc_calc/assets/ancestry/high-LD-regions-hg38-GRCh38.txt",
           "description": "Path to a file that contains areas of high linkage disequilibrium in the reference data (build GRCh38).",
           "format": "file-path",
           "mimetype": "text/plain"
         },
         "ref_format_version": {
           "type": "string",
-          "default": "v0.1"
+          "default": "v0.1",
+          "description": "Version of the default reference database"
         },
         "ancestry_checksums": {
-          "type": "string"
+          "type": "string",
+          "description": "Used to validate files in the reference database when built"
         }
-      },
-      "required": [
-        "geno_ref",
-        "mind_ref",
-        "maf_ref",
-        "hwe_ref",
-        "indep_pairwise_ref",
-        "ld_grch37",
-        "ld_grch38"
-      ]
+      }
     },
-    "compatibility_options": {
-      "title": "Compatibility options",
+    "developer_options": {
+      "title": "Developer options",
       "type": "object",
-      "description": "Define parameters that control how scoring files and target genomes are made compatible with each other",
+      "description": "Control subworkflow execution, useful for debugging",
       "default": "",
       "properties": {
-        "target_build": {
-          "type": "string",
-          "enum": ["GRCh37", "GRCh38"],
-          "description": "Genome build of target genomes"
+        "only_bootstrap": {
+          "type": "boolean",
+          "hidden": true
         },
-        "liftover": {
+        "only_input": {
           "type": "boolean",
-          "description": "Lift scoring files to match your target genomes. Requires build information in the header of the scoring files."
+          "hidden": true
         },
-        "min_lift": {
-          "type": "number",
-          "default": 0.95,
-          "description": "Minimum proportion of variants required to successfully remap a scoring file to a different genome build",
-          "minimum": 0,
-          "maximum": 1
+        "only_compatible": {
+          "type": "boolean",
+          "hidden": true
+        },
+        "only_match": {
+          "type": "boolean",
+          "hidden": true
+        },
+        "only_projection": {
+          "type": "boolean",
+          "hidden": true
+        },
+        "only_score": {
+          "type": "boolean",
+          "hidden": true
+        },
+        "skip_ancestry": {
+          "type": "boolean",
+          "default": true,
+          "hidden": true
         }
-      },
-      "required": ["target_build"]
+      }
     },
-    "matching_options": {
-      "title": "Matching options",
+    "institutional_config_options": {
+      "title": "Institutional config options",
       "type": "object",
-      "description": "Define how variants are matched across scoring files and target genomes",
-      "default": "",
+      "fa_icon": "fas fa-university",
+      "description": "Parameters used to describe centralised config profiles. These should not be edited.",
+      "help_text": "The centralised nf-core configuration profiles use a handful of pipeline parameters to describe themselves. This information is then printed to the Nextflow log when you run a pipeline. You should not need to change these values when you run a pipeline.",
       "properties": {
-        "keep_multiallelic": {
-          "type": "boolean",
-          "description": "Allow matches of scoring file variants to multiallelic variants in the target dataset"
+        "custom_config_version": {
+          "type": "string",
+          "description": "Git commit id for Institutional configs.",
+          "default": "master",
+          "hidden": true,
+          "fa_icon": "fas fa-users-cog"
         },
-        "keep_ambiguous": {
-          "type": "boolean",
-          "description": "Keep matches of scoring file variants to strand ambiguous variants (e.g. A/T and C/G SNPs) in the target dataset. This assumes the scoring file and target dataset report variants on the same strand."
+        "custom_config_base": {
+          "type": "string",
+          "description": "Base directory for Institutional configs.",
+          "default": "https://raw.githubusercontent.com/nf-core/configs/master",
+          "hidden": true,
+          "help_text": "If you're running offline, Nextflow will not be able to fetch the institutional config files from the internet. If you don't need them, then this is not a problem. If you do need them, you should download the files from the repo and tell Nextflow where to find them with this parameter.",
+          "fa_icon": "fas fa-users-cog"
         },
-        "min_overlap": {
-          "type": "number",
-          "default": 0.75,
-          "description": "Minimum proportion of variants present in both the score file and input target genomic data",
-          "fa_icon": "fas fa-cog",
-          "minimum": 0,
-          "maximum": 1
+        "config_profile_name": {
+          "type": "string",
+          "description": "Institutional config name.",
+          "hidden": true,
+          "fa_icon": "fas fa-users-cog"
+        },
+        "config_profile_description": {
+          "type": "string",
+          "description": "Institutional config description.",
+          "hidden": true,
+          "fa_icon": "fas fa-users-cog"
+        },
+        "config_profile_contact": {
+          "type": "string",
+          "description": "Institutional config contact information.",
+          "hidden": true,
+          "fa_icon": "fas fa-users-cog"
+        },
+        "config_profile_url": {
+          "type": "string",
+          "description": "Institutional config URL link.",
+          "hidden": true,
+          "fa_icon": "fas fa-users-cog"
         }
-      },
-      "fa_icon": "fas fa-user-cog"
+      }
     },
     "max_job_request_options": {
       "title": "Max job request options",
@@ -298,7 +389,7 @@
           "description": "Maximum amount of time that can be requested for any single job.",
           "default": "240.h",
           "fa_icon": "far fa-clock",
-          "pattern": "^(\\d+\\.?\\s*(s|m|h|day)\\s*)+$",
+          "pattern": "^(\\d+\\.?\\s*(s|m|h|d|day)\\s*)+$",
           "hidden": true,
           "help_text": "Use to set an upper-limit for the time requirement for each process. Should be a string in the format integer-unit e.g. `--max_time '2.h'`"
         }
@@ -317,6 +408,12 @@
           "fa_icon": "fas fa-question-circle",
           "hidden": true
         },
+        "version": {
+          "type": "boolean",
+          "description": "Display version and exit.",
+          "fa_icon": "fas fa-question-circle",
+          "hidden": true
+        },
         "publish_dir_mode": {
           "type": "string",
           "default": "copy",
@@ -352,88 +449,44 @@
           "description": "Do not use coloured log outputs.",
           "fa_icon": "fas fa-palette",
           "hidden": true
+        },
+        "hook_url": {
+          "type": "string",
+          "description": "Incoming hook URL for messaging service",
+          "fa_icon": "fas fa-people-group",
+          "help_text": "Incoming hook URL for messaging service. Currently, MS Teams and Slack are supported.",
+          "hidden": true
         }
       }
     }
   },
   "allOf": [
     {
-      "$ref": "#/defs/input_output_options"
+      "$ref": "#/definitions/input_output_options"
     },
     {
-      "$ref": "#/defs/ancestry_options"
+      "$ref": "#/definitions/compatibility_options"
     },
     {
-      "$ref": "#/defs/reference_options"
+      "$ref": "#/definitions/new_group_4"
     },
     {
-      "$ref": "#/defs/compatibility_options"
+      "$ref": "#/definitions/genetic_ancestry_options"
     },
     {
-      "$ref": "#/defs/matching_options"
+      "$ref": "#/definitions/reference_options"
     },
     {
-      "$ref": "#/defs/max_job_request_options"
+      "$ref": "#/definitions/developer_options"
     },
     {
-      "$ref": "#/defs/generic_options"
-    }
-  ],
-  "properties": {
-    "only_bootstrap": {
-      "type": "boolean",
-      "hidden": true
-    },
-    "only_input": {
-      "type": "boolean",
-      "hidden": true
-    },
-    "only_compatible": {
-      "type": "boolean",
-      "hidden": true
-    },
-    "only_match": {
-      "type": "boolean",
-      "hidden": true
-    },
-    "only_projection": {
-      "type": "boolean",
-      "hidden": true
+      "$ref": "#/definitions/institutional_config_options"
     },
-    "only_score": {
-      "type": "boolean",
-      "hidden": true
-    },
-    "skip_ancestry": {
-      "type": "boolean",
-      "default": true,
-      "hidden": true
-    },
-    "hook_url": {
-      "type": "string"
-    },
-    "version": {
-      "type": "boolean"
-    },
-    "config_profile_name": {
-      "type": "string"
-    },
-    "config_profile_description": {
-      "type": "string"
-    },
-    "custom_config_version": {
-      "type": "string",
-      "default": "master"
-    },
-    "custom_config_base": {
-      "type": "string",
-      "default": "https://raw.githubusercontent.com/nf-core/configs/master"
-    },
-    "config_profile_contact": {
-      "type": "string"
+    {
+      "$ref": "#/definitions/max_job_request_options"
     },
-    "config_profile_url": {
-      "type": "string"
+    {
+      "$ref": "#/definitions/generic_options"
     }
-  }
+  ]
 }

From 0213dd1a9e8a07367365779e50affd8f66a5f4a0 Mon Sep 17 00:00:00 2001
From: Benjamin Wingfield <bwingfield@ebi.ac.uk>
Date: Mon, 29 Jul 2024 17:15:34 +0100
Subject: [PATCH 13/31] update directory path

---
 docs/how-to/multiple.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/how-to/multiple.rst b/docs/how-to/multiple.rst
index e8f44889..84d98f46 100644
--- a/docs/how-to/multiple.rst
+++ b/docs/how-to/multiple.rst
@@ -133,7 +133,7 @@ Congratulations, you've now calculated multiple scores in parallel!
           combine scores in the PGS Catalog with your own custom scores
 
 After the workflow executes successfully, the calculated scores and a summary
-report should be available in the ``results/make/`` directory by default. If
+report should be available in the ``results/`` directory by default. If
 you're interested in more information, see :ref:`interpret`.
 
 If the workflow didn't execute successfully, have a look at the

From 59a7b63584864ade8a6eadaa2c8ddc98f6ae727c Mon Sep 17 00:00:00 2001
From: smlmbrt <sam.a.lambert@gmail.com>
Date: Mon, 29 Jul 2024 18:32:38 +0100
Subject: [PATCH 14/31] Fix and add schema descriptions.

---
 nextflow_schema.json | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/nextflow_schema.json b/nextflow_schema.json
index aefa1d31..58ab7627 100644
--- a/nextflow_schema.json
+++ b/nextflow_schema.json
@@ -166,13 +166,15 @@
           "default": true,
           "description": "Load allelic frequencies from reference panel when scoring target genomes"
         },
-        "pca_maf_target": {
+        "maf_target": {
           "type": "number",
-          "default": 0.1
+          "default": 0.1,
+          "description": "Minimum MAF threshold in TARGET samples for variants to be included in the PCA."
         },
-        "pca_geno_miss": {
+        "geno_miss_target": {
           "type": "number",
-          "default": 0.1
+          "default": 0.1,
+          "description": "Maximum genotype missingness threshold in TARGET samples for variants to be included in the PCA."
         }
       },
       "required": [

From 18e14e08da03691cfd621789d8211a0ef25baa0b Mon Sep 17 00:00:00 2001
From: smlmbrt <sam.a.lambert@gmail.com>
Date: Mon, 29 Jul 2024 18:39:26 +0100
Subject: [PATCH 15/31] Clarifying changes to PCA variant parameters.

---
 nextflow_schema.json | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/nextflow_schema.json b/nextflow_schema.json
index 58ab7627..b1353129 100644
--- a/nextflow_schema.json
+++ b/nextflow_schema.json
@@ -125,7 +125,7 @@
     "genetic_ancestry_options": {
       "title": "Genetic ancestry options",
       "type": "object",
-      "description": "Parameters used to control genetic ancestry similarity analysis",
+      "description": "Parameters used to control genetic ancestry similarity analysis on TARGET samples and variants included in PCA",
       "default": "",
       "properties": {
         "projection_method": {
@@ -169,12 +169,16 @@
         "maf_target": {
           "type": "number",
           "default": 0.1,
-          "description": "Minimum MAF threshold in TARGET samples for variants to be included in the PCA."
+          "description": "Minimum MAF threshold in TARGET samples for variants to be included in the PCA.",
+          "minimum": 0,
+          "maximum": 1
         },
         "geno_miss_target": {
           "type": "number",
           "default": 0.1,
-          "description": "Maximum genotype missingness threshold in TARGET samples for variants to be included in the PCA."
+          "description": "Maximum genotype missingness threshold in TARGET samples for variants to be included in the PCA.",
+          "minimum": 0,
+          "maximum": 1
         }
       },
       "required": [
@@ -192,7 +196,7 @@
     "reference_options": {
       "title": "Reference options",
       "type": "object",
-      "description": "Define how reference genomes are defined and processed",
+      "description": "Define how genomes and variants in REFERENCE panel are defined and processed for PCA",
       "default": "",
       "properties": {
         "run_ancestry": {
@@ -222,7 +226,7 @@
         "geno_ref": {
           "type": "number",
           "default": 0.1,
-          "description": "Exclude variants with missing call frequencies greater than a threshold (in reference genomes)",
+          "description": "Exclude VARIANTS with percentage of missing genotype calls greater than a threshold (in reference genomes)",
           "minimum": 0,
           "maximum": 1
         },
@@ -231,14 +235,14 @@
           "default": 0.1,
           "minimum": 0,
           "maximum": 1,
-          "description": "Exclude samples with missing call frequencies greater than a threshold (in reference genomes)"
+          "description": "Exclude SAMPLES with percentage of missing genotype calls greater than a threshold (in reference genomes)"
         },
         "maf_ref": {
           "type": "number",
           "default": 0.05,
           "minimum": 0,
           "maximum": 1,
-          "description": "Exclude variants with allele frequency lower than a threshold (in reference genomes)"
+          "description": "Exclude variants with minor allele frequency (MAF) lower than a threshold (in reference genomes)"
         },
         "hwe_ref": {
           "type": "number",

From ad835ae0ea5dbac768486681b6b8fb907a0f5efc Mon Sep 17 00:00:00 2001
From: Benjamin Wingfield <bwingfield@ebi.ac.uk>
Date: Tue, 30 Jul 2024 09:39:40 +0100
Subject: [PATCH 16/31] remove default anaconda channels (#342)

* remove default anaconda channels

* use nodefaults in all conda environment files
---
 conf/modules.config                           | 4 ++--
 environments/fraposa/environment.yml          | 3 ++-
 environments/pgscatalog_utils/environment.yml | 3 ++-
 environments/plink2/environment.yml           | 7 +++++++
 environments/pyyaml/environment.yml           | 4 ++--
 environments/report/environment.yml           | 1 +
 environments/zstd/environment.yml             | 3 ++-
 7 files changed, 18 insertions(+), 7 deletions(-)
 create mode 100644 environments/plink2/environment.yml

diff --git a/conf/modules.config b/conf/modules.config
index 1f345793..b422c3fe 100644
--- a/conf/modules.config
+++ b/conf/modules.config
@@ -43,7 +43,7 @@ process {
     }
 
     withLabel: plink2 {
-        ext.conda = "bioconda::plink2==2.00a5.10"
+        ext.conda = "$projectDir/environments/plink2/environment.yml"
         ext.docker = 'ghcr.io/pgscatalog/plink2'
         ext.singularity = 'oras://ghcr.io/pgscatalog/plink2'
         ext.docker_version = ':2.00a5.10'
@@ -67,7 +67,7 @@ process {
     }
 
     withLabel: pyyaml {
-        ext.conda = "conda-forge::pyyaml==6.0"
+        ext.conda = "$projectDir/environments/pyyaml/environment.yml"
         ext.singularity = 'oras://ghcr.io/pgscatalog/pyyaml'
         ext.singularity_version = ':6.0-singularity'
         ext.docker = 'ghcr.io/pgscatalog/pyyaml'
diff --git a/environments/fraposa/environment.yml b/environments/fraposa/environment.yml
index 34612e75..b69cb02a 100644
--- a/environments/fraposa/environment.yml
+++ b/environments/fraposa/environment.yml
@@ -2,5 +2,6 @@ name: fraposa-pgsc
 channels:
   - conda-forge
   - bioconda
+  - nodefaults  
 dependencies:
-- fraposa-pgsc=1.0.0
\ No newline at end of file
+  - fraposa-pgsc=1.0.0
\ No newline at end of file
diff --git a/environments/pgscatalog_utils/environment.yml b/environments/pgscatalog_utils/environment.yml
index fbf645ed..86c3f114 100644
--- a/environments/pgscatalog_utils/environment.yml
+++ b/environments/pgscatalog_utils/environment.yml
@@ -2,5 +2,6 @@ name: pgscatalog-utils
 channels:
   - conda-forge
   - bioconda
+  - nodefaults  
 dependencies:
-- pgscatalog-utils=1.2.0
+  - pgscatalog-utils=1.2.0
diff --git a/environments/plink2/environment.yml b/environments/plink2/environment.yml
new file mode 100644
index 00000000..1f3bc091
--- /dev/null
+++ b/environments/plink2/environment.yml
@@ -0,0 +1,7 @@
+name: plink2
+channels:
+  - conda-forge
+  - bioconda
+  - nodefaults  
+dependencies: 
+  - plink2==2.00a5.10
\ No newline at end of file
diff --git a/environments/pyyaml/environment.yml b/environments/pyyaml/environment.yml
index 15cb9b73..a7573aff 100644
--- a/environments/pyyaml/environment.yml
+++ b/environments/pyyaml/environment.yml
@@ -2,6 +2,6 @@ name: pyyaml
 channels:
   - conda-forge
   - bioconda
-  - defaults
+  - nodefaults
 dependencies:
-- pyyaml=6.0.1
+  - pyyaml=6.0.1
diff --git a/environments/report/environment.yml b/environments/report/environment.yml
index 9b9bc4b5..43adc80f 100644
--- a/environments/report/environment.yml
+++ b/environments/report/environment.yml
@@ -2,6 +2,7 @@ name: report
 channels:
   - conda-forge
   - bioconda
+  - nodefaults  
 dependencies: 
   - r-jsonlite
   - r-dplyr
diff --git a/environments/zstd/environment.yml b/environments/zstd/environment.yml
index 39242d54..856cc981 100644
--- a/environments/zstd/environment.yml
+++ b/environments/zstd/environment.yml
@@ -2,5 +2,6 @@ name: zstd
 channels:
   - conda-forge
   - bioconda
+  - nodefaults  
 dependencies:
-- zstd=1.4.8
+  - zstd=1.4.8

From f5e507dc415803989cfa00b80471ef8fe6916a60 Mon Sep 17 00:00:00 2001
From: Benjamin Wingfield <bwingfield@ebi.ac.uk>
Date: Tue, 30 Jul 2024 09:42:05 +0100
Subject: [PATCH 17/31] fix group name (matching options)

---
 nextflow_schema.json | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/nextflow_schema.json b/nextflow_schema.json
index b1353129..5b6f74d4 100644
--- a/nextflow_schema.json
+++ b/nextflow_schema.json
@@ -26,7 +26,7 @@
           "format": "directory-path",
           "description": "The output directory where the results will be saved. You have to use absolute paths to storage on Cloud infrastructure.",
           "fa_icon": "fas fa-folder-open",
-          "default": "results"
+          "default": "results/"
         },
         "email": {
           "type": "string",
@@ -99,8 +99,8 @@
       },
       "required": ["target_build"]
     },
-    "new_group_4": {
-      "title": "New Group 4",
+    "matching_options": {
+      "title": "Matching options",
       "type": "object",
       "description": "Define how variants are matched across scoring files and target genomes.",
       "default": "",
@@ -474,7 +474,7 @@
       "$ref": "#/definitions/compatibility_options"
     },
     {
-      "$ref": "#/definitions/new_group_4"
+      "$ref": "#/definitions/matching_options"
     },
     {
       "$ref": "#/definitions/genetic_ancestry_options"

From 351518eb5b8b4f421087088dc67ee202894c6830 Mon Sep 17 00:00:00 2001
From: Benjamin Wingfield <bwingfield@ebi.ac.uk>
Date: Mon, 29 Jul 2024 17:05:17 +0100
Subject: [PATCH 18/31] update schema with support for --maf_target and
 --geno_miss_target

---
 modules/local/ancestry/intersect_variants.nf |   4 +-
 nextflow.config                              |   2 +
 nextflow_schema.json                         | 355 +++++++++++--------
 3 files changed, 208 insertions(+), 153 deletions(-)

diff --git a/modules/local/ancestry/intersect_variants.nf b/modules/local/ancestry/intersect_variants.nf
index efcad0b6..d872bbf4 100644
--- a/modules/local/ancestry/intersect_variants.nf
+++ b/modules/local/ancestry/intersect_variants.nf
@@ -33,8 +33,8 @@ process INTERSECT_VARIANTS {
     pgscatalog-intersect --ref $ref_variants \
         --target $variants \
         --chrom $meta.chrom \
-        --maf_target 0.1 \
-        --geno_miss 0.1 \
+        --maf_target $params.maf_target \
+        --geno_miss $params.geno_miss_target \
         --outdir . \
         -v
 
diff --git a/nextflow.config b/nextflow.config
index 480ab095..27035ba5 100644
--- a/nextflow.config
+++ b/nextflow.config
@@ -43,6 +43,8 @@ params {
     n_popcomp = 5
     normalization_method = "empirical mean mean+var"
     n_normalization = 4
+    maf_target = 0.1
+    geno_miss_target = 0.1
 
     // compatibility params
     liftover = false
diff --git a/nextflow_schema.json b/nextflow_schema.json
index 6caed387..aefa1d31 100644
--- a/nextflow_schema.json
+++ b/nextflow_schema.json
@@ -1,33 +1,49 @@
 {
-  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$schema": "http://json-schema.org/draft-07/schema",
   "$id": "https://raw.githubusercontent.com/pgscatalog/pgsc_calc/master/nextflow_schema.json",
   "title": "pgscatalog/pgsc_calc pipeline parameters",
-  "description": "This pipeline applies scoring files from the PGS Catalog to target set(s) of genotyped samples",
+  "description": "The Polygenic Score Catalog Calculator is a nextflow pipeline for polygenic score calculation",
   "type": "object",
-  "defs": {
+  "definitions": {
     "input_output_options": {
       "title": "Input/output options",
       "type": "object",
       "fa_icon": "fas fa-terminal",
       "description": "Define where the pipeline should find input data and save output data.",
+      "required": ["input", "outdir"],
       "properties": {
         "input": {
           "type": "string",
-          "description": "Path to input samplesheet",
-          "format": "file-path"
+          "format": "file-path",
+          "exists": true,
+          "pattern": "^\\S+\\.(csv|json)$",
+          "description": "Path to comma-separated file containing information about the samples in the experiment.",
+          "help_text": "You will need to create a design file with information about the samples in your experiment before running the pipeline. Use this parameter to specify its location. It has to be a comma-separated file with 3 columns, and a header row.",
+          "fa_icon": "fas fa-file-csv"
+        },
+        "outdir": {
+          "type": "string",
+          "format": "directory-path",
+          "description": "The output directory where the results will be saved. You have to use absolute paths to storage on Cloud infrastructure.",
+          "fa_icon": "fas fa-folder-open",
+          "default": "results"
+        },
+        "email": {
+          "type": "string",
+          "description": "Email address for completion summary.",
+          "fa_icon": "fas fa-envelope",
+          "help_text": "Set this parameter to your e-mail address to get a summary e-mail with details of the run sent to you when the workflow exits. If set in your user config file (`~/.nextflow/config`) then you don't need to specify this on the command line for every run.",
+          "pattern": "^([a-zA-Z0-9_\\-\\.]+)@([a-zA-Z0-9_\\-\\.]+)\\.([a-zA-Z]{2,5})$"
         },
         "format": {
           "type": "string",
           "default": "csv",
-          "fa_icon": "fas fa-cog",
-          "description": "Format of input samplesheet",
-          "enum": ["csv", "json"]
+          "enum": ["csv", "json"],
+          "description": "What format is the samplesheet in? (csv/json)"
         },
         "scorefile": {
           "type": "string",
-          "description": "Path to a scoring file in PGS Catalog format. Multiple scorefiles can be specified using wildcards (e.g., ``--scorefile \"path/to/scores/*.txt\"``)",
-          "fa_icon": "fas fa-file-alt",
-          "format": "file-path"
+          "description": "Path to a scoring file in PGS Catalog format. Multiple scorefiles can be specified using wildcards (e.g., ``--scorefile \\\"path/to/scores/*.txt\\\"``)"
         },
         "pgs_id": {
           "type": "string",
@@ -55,35 +71,68 @@
           "type": "string",
           "description": "Path to a directory that can store relabelled genotypes (and the reference panel intersections and PCA with --run_ancestry) to speed up new PGS calculations on previously harmonized samples",
           "format": "directory-path"
-        },
-        "outdir": {
+        }
+      }
+    },
+    "compatibility_options": {
+      "title": "Compatibility options",
+      "type": "object",
+      "description": "Define parameters that control how scoring files and target genomes are made compatible with each other",
+      "default": "",
+      "properties": {
+        "target_build": {
           "type": "string",
-          "description": "Path to the output directory where the results will be saved.",
-          "fa_icon": "fas fa-folder-open",
-          "format": "directory-path",
-          "default": "results"
+          "description": "Genome build of target genomes",
+          "enum": ["GRCh37", "GRCh38"]
         },
-        "email": {
-          "type": "string",
-          "description": "Email address for completion summary.",
-          "fa_icon": "fas fa-envelope",
-          "help_text": "Set this parameter to your e-mail address to get a summary e-mail with details of the run sent to you when the workflow exits. If set in your user config file (`~/.nextflow/config`) then you don't need to specify this on the command line for every run.",
-          "pattern": "^([a-zA-Z0-9_\\-\\.]+)@([a-zA-Z0-9_\\-\\.]+)\\.([a-zA-Z]{2,5})$"
+        "liftover": {
+          "type": "boolean",
+          "description": "Lift scoring files to match your target genomes. Requires build information in the header of the scoring files."
+        },
+        "min_lift": {
+          "type": "number",
+          "default": 0.95,
+          "description": "Minimum proportion of variants required to successfully remap a scoring file to a different genome build.",
+          "minimum": 0,
+          "maximum": 1
         }
       },
-      "required": ["input", "format"]
+      "required": ["target_build"]
     },
-    "ancestry_options": {
-      "title": "Ancestry options",
+    "new_group_4": {
+      "title": "New Group 4",
       "type": "object",
-      "description": "",
+      "description": "Define how variants are matched across scoring files and target genomes.",
+      "default": "",
+      "properties": {
+        "keep_multiallelic": {
+          "type": "boolean",
+          "description": "Allow matches of scoring file variants to multiallelic variants in the target dataset"
+        },
+        "keep_ambiguous": {
+          "type": "boolean",
+          "description": "Keep matches of scoring file variants to strand ambiguous variants (e.g. A/T and C/G SNPs) in the target dataset. This assumes the scoring file and target dataset report variants on the same strand."
+        },
+        "min_overlap": {
+          "type": "number",
+          "default": 0.75,
+          "minimum": 0,
+          "maximum": 1,
+          "description": "Minimum proportion of variants present in both the score file and input target genomic data"
+        }
+      }
+    },
+    "genetic_ancestry_options": {
+      "title": "Genetic ancestry options",
+      "type": "object",
+      "description": "Parameters used to control genetic ancestry similarity analysis",
       "default": "",
       "properties": {
         "projection_method": {
           "type": "string",
           "default": "oadp",
-          "enum": ["oadp", "sp", "adp"],
-          "description": "The method for PCA prediction. oadp: most accurate. adp: accurate but slow. sp: fast but inaccurate."
+          "description": "The method for PCA prediction. oadp: most accurate. adp: accurate but slow. sp: fast but inaccurate.",
+          "enum": ["oadp", "sp", "adp"]
         },
         "ancestry_method": {
           "type": "string",
@@ -104,8 +153,8 @@
         "normalization_method": {
           "type": "string",
           "default": "empirical mean mean+var",
-          "description": "Method used for normalisation of genetic ancestry",
-          "enum": ["empirical", "mean", "mean+var", "empirical mean mean+var"]
+          "enum": ["empirical", "mean", "mean+var", "empirical mean mean+var"],
+          "description": "Method used for normalisation of genetic ancestry"
         },
         "n_normalization": {
           "type": "integer",
@@ -116,6 +165,14 @@
           "type": "boolean",
           "default": true,
           "description": "Load allelic frequencies from reference panel when scoring target genomes"
+        },
+        "pca_maf_target": {
+          "type": "number",
+          "default": 0.1
+        },
+        "pca_geno_miss": {
+          "type": "number",
+          "default": 0.1
         }
       },
       "required": [
@@ -125,7 +182,9 @@
         "n_popcomp",
         "normalization_method",
         "n_normalization",
-        "load_afreq"
+        "load_afreq",
+        "pca_maf_target",
+        "pca_geno_miss"
       ]
     },
     "reference_options": {
@@ -136,12 +195,13 @@
       "properties": {
         "run_ancestry": {
           "type": "string",
-          "format": "file-path",
-          "description": "Path to reference database. Must be set if --ref_samplesheet is not set."
+          "description": "Path to reference database. Must be set if --ref_samplesheet is not set.",
+          "format": "file-path"
         },
         "ref_samplesheet": {
           "type": "string",
-          "description": "Path to a samplesheet that describes the structure of reference data. Must be set if --ref isn't set."
+          "description": "Path to a samplesheet that describes the structure of reference data. Must be set if --ref isn't set.",
+          "format": "file-path"
         },
         "hg19_chain": {
           "type": "string",
@@ -153,6 +213,7 @@
         "hg38_chain": {
           "type": "string",
           "description": "Path to a UCSC chain file for converting from hg38 to hg19.  Needed if lifting over a custom scoring file.",
+          "pattern": ".*chain.gz$",
           "format": "file-path",
           "mimetype": "application/gzip"
         },
@@ -166,16 +227,16 @@
         "mind_ref": {
           "type": "number",
           "default": 0.1,
-          "description": "Exclude samples with missing call frequencies greater than a threshold (in reference genomes)",
           "minimum": 0,
-          "maximum": 1
+          "maximum": 1,
+          "description": "Exclude samples with missing call frequencies greater than a threshold (in reference genomes)"
         },
         "maf_ref": {
           "type": "number",
           "default": 0.05,
-          "description": "Exclude variants with allele frequency lower than a threshold (in reference genomes)",
           "minimum": 0,
-          "maximum": 1
+          "maximum": 1,
+          "description": "Exclude variants with allele frequency lower than a threshold (in reference genomes)"
         },
         "hwe_ref": {
           "type": "number",
@@ -191,83 +252,113 @@
         },
         "ld_grch37": {
           "type": "string",
+          "default": "/Users/bwingfield/Documents/projects/pgsc_calc/assets/ancestry/high-LD-regions-hg19-GRCh37.txt",
           "description": "Path to a file that contains areas of high linkage disequilibrium in the reference data (build GRCh37).",
           "format": "file-path",
           "mimetype": "text/plain"
         },
         "ld_grch38": {
           "type": "string",
+          "default": "/Users/bwingfield/Documents/projects/pgsc_calc/assets/ancestry/high-LD-regions-hg38-GRCh38.txt",
           "description": "Path to a file that contains areas of high linkage disequilibrium in the reference data (build GRCh38).",
           "format": "file-path",
           "mimetype": "text/plain"
         },
         "ref_format_version": {
           "type": "string",
-          "default": "v0.1"
+          "default": "v0.1",
+          "description": "Version of the default reference database"
         },
         "ancestry_checksums": {
-          "type": "string"
+          "type": "string",
+          "description": "Used to validate files in the reference database when built"
         }
-      },
-      "required": [
-        "geno_ref",
-        "mind_ref",
-        "maf_ref",
-        "hwe_ref",
-        "indep_pairwise_ref",
-        "ld_grch37",
-        "ld_grch38"
-      ]
+      }
     },
-    "compatibility_options": {
-      "title": "Compatibility options",
+    "developer_options": {
+      "title": "Developer options",
       "type": "object",
-      "description": "Define parameters that control how scoring files and target genomes are made compatible with each other",
+      "description": "Control subworkflow execution, useful for debugging",
       "default": "",
       "properties": {
-        "target_build": {
-          "type": "string",
-          "enum": ["GRCh37", "GRCh38"],
-          "description": "Genome build of target genomes"
+        "only_bootstrap": {
+          "type": "boolean",
+          "hidden": true
         },
-        "liftover": {
+        "only_input": {
           "type": "boolean",
-          "description": "Lift scoring files to match your target genomes. Requires build information in the header of the scoring files."
+          "hidden": true
         },
-        "min_lift": {
-          "type": "number",
-          "default": 0.95,
-          "description": "Minimum proportion of variants required to successfully remap a scoring file to a different genome build",
-          "minimum": 0,
-          "maximum": 1
+        "only_compatible": {
+          "type": "boolean",
+          "hidden": true
+        },
+        "only_match": {
+          "type": "boolean",
+          "hidden": true
+        },
+        "only_projection": {
+          "type": "boolean",
+          "hidden": true
+        },
+        "only_score": {
+          "type": "boolean",
+          "hidden": true
+        },
+        "skip_ancestry": {
+          "type": "boolean",
+          "default": true,
+          "hidden": true
         }
-      },
-      "required": ["target_build"]
+      }
     },
-    "matching_options": {
-      "title": "Matching options",
+    "institutional_config_options": {
+      "title": "Institutional config options",
       "type": "object",
-      "description": "Define how variants are matched across scoring files and target genomes",
-      "default": "",
+      "fa_icon": "fas fa-university",
+      "description": "Parameters used to describe centralised config profiles. These should not be edited.",
+      "help_text": "The centralised nf-core configuration profiles use a handful of pipeline parameters to describe themselves. This information is then printed to the Nextflow log when you run a pipeline. You should not need to change these values when you run a pipeline.",
       "properties": {
-        "keep_multiallelic": {
-          "type": "boolean",
-          "description": "Allow matches of scoring file variants to multiallelic variants in the target dataset"
+        "custom_config_version": {
+          "type": "string",
+          "description": "Git commit id for Institutional configs.",
+          "default": "master",
+          "hidden": true,
+          "fa_icon": "fas fa-users-cog"
         },
-        "keep_ambiguous": {
-          "type": "boolean",
-          "description": "Keep matches of scoring file variants to strand ambiguous variants (e.g. A/T and C/G SNPs) in the target dataset. This assumes the scoring file and target dataset report variants on the same strand."
+        "custom_config_base": {
+          "type": "string",
+          "description": "Base directory for Institutional configs.",
+          "default": "https://raw.githubusercontent.com/nf-core/configs/master",
+          "hidden": true,
+          "help_text": "If you're running offline, Nextflow will not be able to fetch the institutional config files from the internet. If you don't need them, then this is not a problem. If you do need them, you should download the files from the repo and tell Nextflow where to find them with this parameter.",
+          "fa_icon": "fas fa-users-cog"
         },
-        "min_overlap": {
-          "type": "number",
-          "default": 0.75,
-          "description": "Minimum proportion of variants present in both the score file and input target genomic data",
-          "fa_icon": "fas fa-cog",
-          "minimum": 0,
-          "maximum": 1
+        "config_profile_name": {
+          "type": "string",
+          "description": "Institutional config name.",
+          "hidden": true,
+          "fa_icon": "fas fa-users-cog"
+        },
+        "config_profile_description": {
+          "type": "string",
+          "description": "Institutional config description.",
+          "hidden": true,
+          "fa_icon": "fas fa-users-cog"
+        },
+        "config_profile_contact": {
+          "type": "string",
+          "description": "Institutional config contact information.",
+          "hidden": true,
+          "fa_icon": "fas fa-users-cog"
+        },
+        "config_profile_url": {
+          "type": "string",
+          "description": "Institutional config URL link.",
+          "hidden": true,
+          "fa_icon": "fas fa-users-cog"
         }
-      },
-      "fa_icon": "fas fa-user-cog"
+      }
     },
     "max_job_request_options": {
       "title": "Max job request options",
@@ -298,7 +389,7 @@
           "description": "Maximum amount of time that can be requested for any single job.",
           "default": "240.h",
           "fa_icon": "far fa-clock",
-          "pattern": "^(\\d+\\.?\\s*(s|m|h|day)\\s*)+$",
+          "pattern": "^(\\d+\\.?\\s*(s|m|h|d|day)\\s*)+$",
           "hidden": true,
           "help_text": "Use to set an upper-limit for the time requirement for each process. Should be a string in the format integer-unit e.g. `--max_time '2.h'`"
         }
@@ -317,6 +408,12 @@
           "fa_icon": "fas fa-question-circle",
           "hidden": true
         },
+        "version": {
+          "type": "boolean",
+          "description": "Display version and exit.",
+          "fa_icon": "fas fa-question-circle",
+          "hidden": true
+        },
         "publish_dir_mode": {
           "type": "string",
           "default": "copy",
@@ -352,88 +449,44 @@
           "description": "Do not use coloured log outputs.",
           "fa_icon": "fas fa-palette",
           "hidden": true
+        },
+        "hook_url": {
+          "type": "string",
+          "description": "Incoming hook URL for messaging service",
+          "fa_icon": "fas fa-people-group",
+          "help_text": "Incoming hook URL for messaging service. Currently, MS Teams and Slack are supported.",
+          "hidden": true
         }
       }
     }
   },
   "allOf": [
     {
-      "$ref": "#/defs/input_output_options"
+      "$ref": "#/definitions/input_output_options"
     },
     {
-      "$ref": "#/defs/ancestry_options"
+      "$ref": "#/definitions/compatibility_options"
     },
     {
-      "$ref": "#/defs/reference_options"
+      "$ref": "#/definitions/new_group_4"
     },
     {
-      "$ref": "#/defs/compatibility_options"
+      "$ref": "#/definitions/genetic_ancestry_options"
     },
     {
-      "$ref": "#/defs/matching_options"
+      "$ref": "#/definitions/reference_options"
     },
     {
-      "$ref": "#/defs/max_job_request_options"
+      "$ref": "#/definitions/developer_options"
     },
     {
-      "$ref": "#/defs/generic_options"
-    }
-  ],
-  "properties": {
-    "only_bootstrap": {
-      "type": "boolean",
-      "hidden": true
-    },
-    "only_input": {
-      "type": "boolean",
-      "hidden": true
-    },
-    "only_compatible": {
-      "type": "boolean",
-      "hidden": true
-    },
-    "only_match": {
-      "type": "boolean",
-      "hidden": true
-    },
-    "only_projection": {
-      "type": "boolean",
-      "hidden": true
+      "$ref": "#/definitions/institutional_config_options"
     },
-    "only_score": {
-      "type": "boolean",
-      "hidden": true
-    },
-    "skip_ancestry": {
-      "type": "boolean",
-      "default": true,
-      "hidden": true
-    },
-    "hook_url": {
-      "type": "string"
-    },
-    "version": {
-      "type": "boolean"
-    },
-    "config_profile_name": {
-      "type": "string"
-    },
-    "config_profile_description": {
-      "type": "string"
-    },
-    "custom_config_version": {
-      "type": "string",
-      "default": "master"
-    },
-    "custom_config_base": {
-      "type": "string",
-      "default": "https://raw.githubusercontent.com/nf-core/configs/master"
-    },
-    "config_profile_contact": {
-      "type": "string"
+    {
+      "$ref": "#/definitions/max_job_request_options"
     },
-    "config_profile_url": {
-      "type": "string"
+    {
+      "$ref": "#/definitions/generic_options"
     }
-  }
+  ]
 }

From f9c462490bfa24f91c685b6df5fc9ea0ff0d347f Mon Sep 17 00:00:00 2001
From: Benjamin Wingfield <bwingfield@ebi.ac.uk>
Date: Tue, 30 Jul 2024 10:39:05 +0100
Subject: [PATCH 19/31] bump version

---
 nextflow.config | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nextflow.config b/nextflow.config
index 27035ba5..d18f0d3e 100644
--- a/nextflow.config
+++ b/nextflow.config
@@ -258,7 +258,7 @@ manifest {
     description     = 'The Polygenic Score Catalog Calculator is a nextflow pipeline for polygenic score calculation'
     mainScript      = 'main.nf'
     nextflowVersion = '>=23.10.0'
-    version         = '2.0.0-beta.1'
+    version         = '2.0.0-beta.2'
 }
 
 // Load modules.config for DSL2 module specific options

From 34539187afa82bd3b880f5180063d89f8f353835 Mon Sep 17 00:00:00 2001
From: Benjamin Wingfield <bwingfield@ebi.ac.uk>
Date: Tue, 30 Jul 2024 12:01:34 +0100
Subject: [PATCH 20/31] docs review

---
 docs/conf.py                     |   2 +-
 docs/explanation/match.rst       |   4 +-
 docs/explanation/output.rst      |   2 +
 docs/how-to/bigjob.rst           | 162 ++++++++++++++++++++++++-------
 docs/how-to/cache.rst            |  25 ++---
 docs/how-to/calculate_custom.rst |   2 +-
 docs/how-to/offline.rst          |   8 +-
 docs/how-to/prepare.rst          |   2 +
 docs/how-to/samplesheet.rst      |   2 +-
 docs/index.rst                   |   7 +-
 10 files changed, 162 insertions(+), 54 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index 86057e0a..8e6ac019 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -22,7 +22,7 @@
 
 project = 'Polygenic Score (PGS) Catalog Calculator'
 copyright = 'Polygenic Score (PGS) Catalog team (licensed under Apache License V2)'
-# author = 'Polygenic Score (PGS) Catalog team'
+author = 'Polygenic Score (PGS) Catalog team'
 
 
 # -- General configuration ---------------------------------------------------
diff --git a/docs/explanation/match.rst b/docs/explanation/match.rst
index a072eb65..85449b4c 100644
--- a/docs/explanation/match.rst
+++ b/docs/explanation/match.rst
@@ -37,6 +37,8 @@ When you evaluate the predictive performance of a score with low match rates it
 
 If you reduce ``--min_overlap`` then the calculator will output scores calculated with the remaining variants, **but these scores may not be representative of the original data submitted to the PGS Catalog.**
 
+.. _wgs:
+
 Are your target genomes imputed? Are they WGS?
 ----------------------------------------------
 
@@ -49,7 +51,7 @@ In the future we plan to improve support for WGS.
 Did you set the correct genome build?
 -------------------------------------
 
-The calculator will automatically grab scoring files in the correct genome build from the PGS Catalog. If match rates are low it may be because you have specified the wrong genome build. If you're using custom scoring files and the match rate is low it is possible that the `--liftover` command may have been omitted. 
+The calculator will automatically grab scoring files in the correct genome build from the PGS Catalog. If match rates are low it may be because you have specified the wrong genome build. If you're using custom scoring files and the match rate is low it is possible that the ``--liftover`` command may have been omitted. 
 
 I'm still getting match rate errors. How do I figure out what's wrong?
 ----------------------------------------------------------------------
diff --git a/docs/explanation/output.rst b/docs/explanation/output.rst
index 4aff03fe..37324e12 100644
--- a/docs/explanation/output.rst
+++ b/docs/explanation/output.rst
@@ -23,6 +23,7 @@ Calculated scores are stored in a gzipped-text space-delimted text file called
 seperate row (``length = n_samples*n_pgs``), and there will be at least four columns with the following headers:
 
 - ``sampleset``: the name of the input sampleset, or ``reference`` for the panel.
+- ``FID``: the family identifier of each sample within the dataset (may be the same as IID).
 - ``IID``: the identifier of each sample within the dataset.
 - ``PGS``: the accession ID of the PGS being reported.
 - ``SUM``: reports the weighted sum of *effect_allele* dosages multiplied by their *effect_weight*
@@ -56,6 +57,7 @@ describing the analysis of the target samples in relation to the reference panel
 following headers:
 
 - ``sampleset``: the name of the input sampleset, or ``reference`` for the panel.
+- ``FID``: the family identifier of each sample within the dataset (may be the same as IID).
 - ``IID``: the identifier of each sample within the dataset.
 - ``[PC1 ... PCN]``: The projection of the sample within the PCA space defined by the reference panel. There will be as
   many PC columns as there are PCs calculated (default: 10).
diff --git a/docs/how-to/bigjob.rst b/docs/how-to/bigjob.rst
index 68380088..8940b616 100644
--- a/docs/how-to/bigjob.rst
+++ b/docs/how-to/bigjob.rst
@@ -74,43 +74,132 @@ limits.
 .. warning:: You'll probably want to use ``-profile singularity`` on a HPC. The
           pipeline requires Singularity v3.7 minimum.
    
-However, in general you will have to adjust the ``executor`` options and job resource
-allocations (e.g. ``process_low``). Here's an example for an LSF cluster:
+Here's an example configuration running about 100 scores in parallel
+on UK Biobank with a SLURM cluster:
 
 .. code-block:: text
 
     process {
-        queue = 'short'
-        clusterOptions = ''
-        scratch = true
+        errorStrategy = 'retry'
+        maxRetries = 3
+        maxErrors = '-1'
+        executor = 'slurm'
+
+        withName: 'DOWNLOAD_SCOREFILES' {
+          cpus = 1
+          memory = { 1.GB * task.attempt }
+          time = { 1.hour * task.attempt }
+        }
 
-        withLabel:process_low {
-            cpus   = 2
-            memory = 8.GB
-            time   = 1.h
+        withName: 'COMBINE_SCOREFILES' {
+          cpus = 1
+          memory = { 8.GB * task.attempt }
+          time = { 2.hour * task.attempt }
         }
-        withLabel:process_medium {
-            cpus   = 8
-            memory = 64.GB
-            time   = 4.h
+
+        withName: 'PLINK2_MAKEBED' {
+          cpus = 2
+          memory = { 8.GB * task.attempt }
+          time = { 1.hour * task.attempt }
         }
-    }
 
-    executor {
-        name = 'lsf'
-        jobName = { "$task.hash" }
-    } 
+        withName: 'RELABEL_IDS' {
+          cpus = 1
+          memory = { 16.GB * task.attempt }
+          time = { 1.hour * task.attempt }
+        }
+
+        withName: 'PLINK2_ORIENT' {
+          cpus = 2
+          memory = { 8.GB * task.attempt }
+          time = { 1.hour * task.attempt }
+        }
+
+        withName: 'DUMPSOFTWAREVERSIONS' {
+          cpus = 1
+          memory = { 1.GB * task.attempt }
+          time = { 1.hour * task.attempt }
+        }
+
+        withName: 'ANCESTRY_ANALYSIS' {
+          cpus = { 1 * task.attempt }
+          memory = { 8.GB * task.attempt }
+          time = { 1.hour * task.attempt }
+        }
+
+        withName: 'SCORE_REPORT' {
+          cpus = 2
+          memory = { 8.GB * task.attempt }
+          time = { 1.hour * task.attempt }
+        }
 
-In SLURM, queue is equivalent to a partition. Specific cluster parameters can be
-provided by modifying ``clusterOptions``. You should change ``cpus``,
-``memory``, and ``time`` to match the amount of resources used. Assuming the
-configuration file you set up is saved as ``my_custom.config`` in your current
-working directory, you're ready to run pgsc_calc. Instead of running nextflow
-directly on the shell, save a bash script (``run_pgscalc.sh``) to a file
-instead:
+        withName: 'EXTRACT_DATABASE' {
+          cpus = 1
+          memory = { 8.GB * task.attempt }
+          time = { 1.hour * task.attempt }
+        }
+
+        withName: 'PLINK2_RELABELPVAR' {
+          cpus = 2
+          memory = { 16.GB * task.attempt }
+          time = { 2.hour * task.attempt }
+        }
+
+        withName: 'INTERSECT_VARIANTS' {
+          cpus = 2
+          memory = { 8.GB * task.attempt }
+          time = { 1.hour * task.attempt }
+        }
+
+        withName: 'MATCH_VARIANTS' {
+          cpus = 2
+          memory = { 32.GB * task.attempt }
+          time = { 6.hour * task.attempt }
+        }
+
+        withName: 'FILTER_VARIANTS' {
+          cpus = 2
+          memory = { 16.GB * task.attempt }
+          time = { 1.hour * task.attempt }
+        }
+
+        withName: 'MATCH_COMBINE' {
+          cpus = 4
+          memory = { 64.GB * task.attempt }
+          time = { 6.hour * task.attempt }
+        }
+
+        withName: 'FRAPOSA_PCA' {
+          cpus = 2
+          memory = { 8.GB * task.attempt }
+          time = { 1.hour * task.attempt }
+        }
+
+        withName: 'PLINK2_SCORE' {
+          cpus = 2
+          memory = { 8.GB * task.attempt }
+          time = { 12.hour * task.attempt }
+        }
+
+        withName: 'SCORE_AGGREGATE' {
+          cpus = 2
+          memory = { 16.GB * task.attempt }
+          time = { 4.hour * task.attempt }
+        }
+    }
+
+Assuming the configuration file you set up is saved as
+``my_custom.config`` in your current working directory, you're ready
+to run pgsc_calc. Instead of running nextflow directly on the shell,
+save a bash script (``run_pgscalc.sh``) to a file instead:
 
 .. code-block:: bash
-                
+
+    #SBATCH -J ukbiobank_pgs
+    #SBATCH -c 1
+    #SBATCH -t 24:00:00
+    #SBATCH --mem=2G
+    
     export NXF_ANSI_LOG=false
     export NXF_OPTS="-Xms500M -Xmx2G" 
     
@@ -126,20 +215,23 @@ instead:
 .. note:: The name of the nextflow and singularity modules will be different in
           your local environment
 
-          .. warning:: Make sure to copy input data to fast storage, and run the pipeline
-            on the same fast storage area. You might include these steps in your
-            bash script. Ask your sysadmin for help if you're not sure what this
-            means.
+.. warning:: Make sure to copy input data to fast storage, and run the
+            pipeline on the same fast storage area. You might include
+            these steps in your bash script. Ask your sysadmin for
+            help if you're not sure what this means.
           
 .. code-block:: console
             
-    $ bsub -M 2GB -q short -o output.txt < run_pgscalc.sh
-
+    $ sbatch run_pgsc_calc.sh
+    
 This will submit a nextflow driver job, which will submit additional jobs for
-each process in the workflow. The nextflow driver requires up to 4GB of RAM
-(bsub's ``-M`` parameter) and 2 CPUs to use (see a guide for `HPC users`_ here).
+each process in the workflow. The nextflow driver requires up to 4GB of RAM and 2 CPUs to use (see a guide for `HPC users`_ here).
 
-.. _`LSF and PBS`: https://nextflow.io/docs/latest/executor.html#slurm
 .. _`HPC users`: https://www.nextflow.io/blog/2021/5_tips_for_hpc_users.html
 .. _`a nextflow profile`: https://github.com/nf-core/configs
 
+
+Cloud deployments
+-----------------
+
+We've deployed the calculator to Google Cloud Batch but some :doc:`special configuration is required<cloud>`.
diff --git a/docs/how-to/cache.rst b/docs/how-to/cache.rst
index 6cad3a0b..b4f08697 100644
--- a/docs/how-to/cache.rst
+++ b/docs/how-to/cache.rst
@@ -1,23 +1,26 @@
 .. _cache:
 
-How do I speed up `pgsc_calc` computation times and avoid re-running code?
-==========================================================================
+How do I speed up computation times and avoid re-running code?
+==============================================================
 
-If you intend to run `pgsc_calc` multiple times on the same target samples (e.g.
+If you intend to run ``pgsc_calc`` multiple times on the same target samples (e.g.
 on different sets of PGS, with different variant matching flags) it is worth cacheing
 information on invariant steps of the pipeline:
 
 - Genotype harmonzation (variant relabeling steps)
-- Steps of `--run_ancestry` that: match variants between the target and reference panel and
+- Steps of ``--run_ancestry`` that: match variants between the target and reference panel and
   generate PCA loadings that can be used to adjust the PGS for ancestry.
 
-To do this you must specify a directory that can store these information across runs using the
-`--genotypes_cache` flag to the nextflow command (also see :ref:`param ref`). Future runs of the
-pipeline that use the same cache directory should then skip these steps and proceed to run only the
-steps needed to calculate new PGS. This is slightly different than using the `-resume command in
-nextflow <https://www.nextflow.io/blog/2019/demystifying-nextflow-resume.html>`_ which mainly checks the
-`work` directory and is more often used for restarting the pipeline when a specific step has failed
-(e.g. for exceeding memory limits).
+To do this you must specify a directory that can store these
+information across runs using the ``--genotypes_cache`` flag to the
+nextflow command (also see :ref:`param ref`). Future runs of the
+pipeline that use the same cache directory should then skip these
+steps and proceed to run only the steps needed to calculate new PGS.
+This is slightly different than using the `-resume command in nextflow
+<https://www.nextflow.io/blog/2019/demystifying-nextflow-resume.html>`_
+which mainly checks the ``work`` directory and is more often used for
+restarting the pipeline when a specific step has failed (e.g. for
+exceeding memory limits).
 
 .. warning:: Always use a new cache directory for different samplesets, as redundant names may clash across runs. 
 
diff --git a/docs/how-to/calculate_custom.rst b/docs/how-to/calculate_custom.rst
index 77333dee..5d7f17b4 100644
--- a/docs/how-to/calculate_custom.rst
+++ b/docs/how-to/calculate_custom.rst
@@ -26,7 +26,7 @@ minimal header in the following format:
 Header::
 
     #pgs_name=metaGRS_CAD
-    #pgs_name=metaGRS_CAD    
+    #pgs_id=metaGRS_CAD    
     #trait_reported=Coronary artery disease
     #genome_build=GRCh37
 
diff --git a/docs/how-to/offline.rst b/docs/how-to/offline.rst
index a77bf118..ca9e8da4 100644
--- a/docs/how-to/offline.rst
+++ b/docs/how-to/offline.rst
@@ -127,8 +127,12 @@ panel too. See :ref:`norm`.
 Download scoring files
 ----------------------
 
-It's best to manually download scoring files from the PGS Catalog in the correct
-genome build. Using PGS001229 as an example:
+.. tip:: Use our CLI application ``pgscatalog-download`` to `download multiple scoring`_ files in parallel and the correct genome build
+
+.. _download multiple scoring: https://pygscatalog.readthedocs.io/en/latest/how-to/guides/download.html
+
+You'll need to preload scoring files in the correct genome build.
+Using PGS001229 as an example:
 
 https://ftp.ebi.ac.uk/pub/databases/spot/pgs/scores/PGS001229/ScoringFiles/
 
diff --git a/docs/how-to/prepare.rst b/docs/how-to/prepare.rst
index d74427bc..ec174c4f 100644
--- a/docs/how-to/prepare.rst
+++ b/docs/how-to/prepare.rst
@@ -52,6 +52,8 @@ VCF from WGS
 See https://github.com/PGScatalog/pgsc_calc/discussions/123 for discussion about tools
 to convert the VCF files into ones suitable for calculating PGS.
 
+If you input WGS data to the calculator without following the steps above then you will probably encounter match rate errors. For more information, see: :ref:`wgs`
+
 
 ``plink`` binary fileset (bfile)
 --------------------------------
diff --git a/docs/how-to/samplesheet.rst b/docs/how-to/samplesheet.rst
index ede58d8b..f94a84e6 100644
--- a/docs/how-to/samplesheet.rst
+++ b/docs/how-to/samplesheet.rst
@@ -27,7 +27,7 @@ download here <../../assets/examples/samplesheet.csv>`.
 
 There are four mandatory columns:
 
-- **sampleset**: A text string (no spaces, or reserved characters [ '.' or '_' ]) referring
+- **sampleset**: A text string (no spaces, or reserved characters [ ``.`` or ``_`` ]) referring
   to the name of a :term:`target dataset` of genotyping data containing at least one
   sample/individual (however cohort datasets will often contain many individuals with
   combined genotyped/imputed data). Data from a sampleset may be input as a single file,
diff --git a/docs/index.rst b/docs/index.rst
index dca0cb76..bec94718 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -54,7 +54,7 @@ The workflow relies on open source scientific software, including:
 A full description of included software is described in :ref:`containers`.
 
 .. _PLINK 2: https://www.cog-genomics.org/plink/2.0/
-.. _PGS Catalog Utilities: https://github.com/PGScatalog/pgscatalog_utils
+.. _PGS Catalog Utilities: https://github.com/PGScatalog/pygscatalog
 .. _FRAPOSA: https://github.com/PGScatalog/fraposa_pgsc
 
 
@@ -120,7 +120,10 @@ Documentation
 Changelog
 ---------
 
-The :doc:`Changelog page<changelog>` describes fixes and enhancements for each version.
+The `Changelog page`_ describes fixes and enhancements for each version.
+
+.. _`Changelog page`: https://github.com/PGScatalog/pgsc_calc/releases
+
 
 Features under development
 --------------------------

From 23551621c8191ad4b9472681ab224a5c79c67316 Mon Sep 17 00:00:00 2001
From: smlmbrt <sam.a.lambert@gmail.com>
Date: Tue, 30 Jul 2024 14:05:52 +0100
Subject: [PATCH 21/31] Documenting new target MAF/missingness variant filters
 in explainer.

---
 docs/explanation/geneticancestry.rst | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/docs/explanation/geneticancestry.rst b/docs/explanation/geneticancestry.rst
index fe8660f5..e703f6ab 100644
--- a/docs/explanation/geneticancestry.rst
+++ b/docs/explanation/geneticancestry.rst
@@ -130,7 +130,15 @@ how-to guide), and has the following steps:
         for variant-level QC (SNPs in Hardy–Weinberg equilibrium [p > 1e-04] that are bi-allelic and non-ambiguous,
         with low missingness [<10%], and minor allele frequency [MAF > 5%]) and sample-quality (missingness <10%).
         LD-pruning is then applied to the variants and sample passing these checks (r\ :sup:`2` threshold = 0.05), excluding
-        complex regions with high LD (e.g. MHC). These methods are implemented in the ``FILTER_VARIANTS`` module.
+        complex regions with high LD (e.g. MHC). These methods are implemented in the ``FILTER_VARIANTS`` module, and
+        the default settings can be changed (see :doc:`schema (Reference options) <params>`).
+
+        1.  **Additional variant filters on TARGET samples**: in ``v2.0.0-beta`` we introduced the ability to filter
+            target sample variants using minimum MAF [default 10%] and maximum genotype missingness [default 10%] to
+            improve PCA robustness when using imputed genotype data (see :doc:`schema (Ancestry options) <params>`).
+            *Note: these parameters may need to be adjusted depending on your input data (currently optimized for large
+            cohorts like UKB), for individual samples we recommend the MAF filter to be lowered (``--pca_maf_target 0``)
+            to ensure homozygous reference calls are included.*
 
     2.  **PCA**: the LD-pruned variants of the unrelated samples passing QC are then used to define the PCA space of the
         reference panel (default: 10 PCs) using `FRAPOSA`_ (Fast and Robust Ancestry Prediction by using Online singular

From d9bca8b0079c2e24c92135f5ec7b3f0c63165e43 Mon Sep 17 00:00:00 2001
From: Benjamin Wingfield <bwingfield@ebi.ac.uk>
Date: Tue, 30 Jul 2024 15:10:43 +0100
Subject: [PATCH 22/31] remove biocontainers from citations

---
 CITATIONS.md | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/CITATIONS.md b/CITATIONS.md
index 128445bc..7cba0759 100644
--- a/CITATIONS.md
+++ b/CITATIONS.md
@@ -33,9 +33,6 @@
 * [Bioconda](https://pubmed.ncbi.nlm.nih.gov/29967506/)
     > Grüning B, Dale R, Sjödin A, Chapman BA, Rowe J, Tomkins-Tinch CH, Valieris R, Köster J; Bioconda Team. Bioconda: sustainable and comprehensive software distribution for the life sciences. Nat Methods. 2018 Jul;15(7):475-476. doi: 10.1038/s41592-018-0046-7. PubMed PMID: 29967506.
 
-* [BioContainers](https://pubmed.ncbi.nlm.nih.gov/28379341/)
-    > da Veiga Leprevost F, Grüning B, Aflitos SA, Röst HL, Uszkoreit J, Barsnes H, Vaudel M, Moreno P, Gatto L, Weber J, Bai M, Jimenez RC, Sachsenberg T, Pfeuffer J, Alvarez RV, Griss J, Nesvizhskii AI, Perez-Riverol Y. BioContainers: an open-source and community-driven framework for software standardization. Bioinformatics. 2017 Aug 15;33(16):2580-2582. doi: 10.1093/bioinformatics/btx192. PubMed PMID: 28379341; PubMed Central PMCID: PMC5870671.
-
 * [Docker](https://dl.acm.org/doi/10.5555/2600239.2600241)
 
 * [Singularity](https://pubmed.ncbi.nlm.nih.gov/28494014/)

From fbe96fafb2df0ca8d1a6315b41032d4ca0771f32 Mon Sep 17 00:00:00 2001
From: Benjamin Wingfield <bwingfield@ebi.ac.uk>
Date: Tue, 30 Jul 2024 15:18:01 +0100
Subject: [PATCH 23/31] update link from pgscatalog_utils to pygscatalog

---
 docs/_templates/globaltoc.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/_templates/globaltoc.html b/docs/_templates/globaltoc.html
index 048f9eef..b7cad074 100644
--- a/docs/_templates/globaltoc.html
+++ b/docs/_templates/globaltoc.html
@@ -35,7 +35,7 @@ <h3>Useful links</h3>
       <li><a href="https://github.com/pgscatalog/pgsc_calc/issues">Issue tracker</a></li>
       <li><a href="https://github.com/PGScatalog/pgsc_calc/discussions">Discussion board</a></li>
   </ul>
-  <li><a href="https://github.com/PGScatalog/pgscatalog_utils">pgscatalog_utils Github</a></li>
+  <li><a href="https://github.com/PGScatalog/pygscatalog">pgscatalog-utils GitHub</a></li>
 </ul>
 
 <hr>

From 94dadff221c819cf99347b263cd66daec343650b Mon Sep 17 00:00:00 2001
From: smlmbrt <sam.a.lambert@gmail.com>
Date: Tue, 30 Jul 2024 17:58:31 +0100
Subject: [PATCH 24/31] Re-align param names and module with schema

---
 modules/local/ancestry/intersect_variants.nf | 4 ++--
 nextflow_schema.json                         | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/modules/local/ancestry/intersect_variants.nf b/modules/local/ancestry/intersect_variants.nf
index d872bbf4..e5c2efe3 100644
--- a/modules/local/ancestry/intersect_variants.nf
+++ b/modules/local/ancestry/intersect_variants.nf
@@ -33,8 +33,8 @@ process INTERSECT_VARIANTS {
     pgscatalog-intersect --ref $ref_variants \
         --target $variants \
         --chrom $meta.chrom \
-        --maf_target $params.maf_target \
-        --geno_miss $params.geno_miss_target \
+        --maf_target $params.pca_maf_target \
+        --geno_miss $params.pca_geno_miss_target \
         --outdir . \
         -v
 
diff --git a/nextflow_schema.json b/nextflow_schema.json
index 18a88128..174ba51b 100644
--- a/nextflow_schema.json
+++ b/nextflow_schema.json
@@ -166,14 +166,14 @@
           "default": true,
           "description": "Load allelic frequencies from reference panel when scoring target genomes"
         },
-        "maf_target": {
+        "pca_maf_target": {
           "type": "number",
           "default": 0.1,
           "description": "Minimum MAF threshold in TARGET samples for variants to be included in the PCA.",
           "minimum": 0,
           "maximum": 1
         },
-        "geno_miss_target": {
+        "pca_geno_miss_target": {
           "type": "number",
           "default": 0.1,
           "description": "Maximum genotype missingness threshold in TARGET samples for variants to be included in the PCA.",
@@ -190,7 +190,7 @@
         "n_normalization",
         "load_afreq",
         "pca_maf_target",
-        "pca_geno_miss"
+        "pca_geno_miss_target"
       ]
     },
     "reference_options": {

From 70471cf7c411865defa3933cf36e8316eceabc69 Mon Sep 17 00:00:00 2001
From: smlmbrt <sam.a.lambert@gmail.com>
Date: Tue, 30 Jul 2024 18:06:58 +0100
Subject: [PATCH 25/31] Add to conf

---
 nextflow.config | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/nextflow.config b/nextflow.config
index d18f0d3e..c4abafc8 100644
--- a/nextflow.config
+++ b/nextflow.config
@@ -43,8 +43,8 @@ params {
     n_popcomp = 5
     normalization_method = "empirical mean mean+var"
     n_normalization = 4
-    maf_target = 0.1
-    geno_miss_target = 0.1
+    pca_maf_target = 0.1
+    pca_geno_miss_target = 0.1
 
     // compatibility params
     liftover = false

From a2dc50ac0eb40e33c3389b2c3c545abeb2786b5a Mon Sep 17 00:00:00 2001
From: Benjamin Wingfield <bwingfield@ebi.ac.uk>
Date: Wed, 31 Jul 2024 10:48:21 +0100
Subject: [PATCH 26/31] Update gitignore

---
 .gitignore | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitignore b/.gitignore
index a595dcfe..96198474 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,3 +11,5 @@ assets/report/renv/
 assets/report/report.Rproj
 .Rprofile
 tests/.venv/
+docs/.venv/
+docs/_build
\ No newline at end of file

From 11f12b8fcb99493529f9c425289cd30deb54dcda Mon Sep 17 00:00:00 2001
From: smlmbrt <sam.a.lambert@gmail.com>
Date: Wed, 31 Jul 2024 10:52:25 +0100
Subject: [PATCH 27/31] Don't filter TARGET variants for MAF by default, the
 missingness filter remains the same.

---
 docs/explanation/geneticancestry.rst | 6 ++----
 nextflow.config                      | 2 +-
 nextflow_schema.json                 | 2 +-
 3 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/docs/explanation/geneticancestry.rst b/docs/explanation/geneticancestry.rst
index e703f6ab..c1176559 100644
--- a/docs/explanation/geneticancestry.rst
+++ b/docs/explanation/geneticancestry.rst
@@ -134,11 +134,9 @@ how-to guide), and has the following steps:
         the default settings can be changed (see :doc:`schema (Reference options) <params>`).
 
         1.  **Additional variant filters on TARGET samples**: in ``v2.0.0-beta`` we introduced the ability to filter
-            target sample variants using minimum MAF [default 10%] and maximum genotype missingness [default 10%] to
+            target sample variants using a maximum genotype missingness [default 10%] and/or minimum MAF [default 0%] to
             improve PCA robustness when using imputed genotype data (see :doc:`schema (Ancestry options) <params>`).
-            *Note: these parameters may need to be adjusted depending on your input data (currently optimized for large
-            cohorts like UKB), for individual samples we recommend the MAF filter to be lowered (``--pca_maf_target 0``)
-            to ensure homozygous reference calls are included.*
+            *Note: these parameters may need to be adjusted depending on your input data.*
 
     2.  **PCA**: the LD-pruned variants of the unrelated samples passing QC are then used to define the PCA space of the
         reference panel (default: 10 PCs) using `FRAPOSA`_ (Fast and Robust Ancestry Prediction by using Online singular
diff --git a/nextflow.config b/nextflow.config
index c4abafc8..af27b640 100644
--- a/nextflow.config
+++ b/nextflow.config
@@ -43,7 +43,7 @@ params {
     n_popcomp = 5
     normalization_method = "empirical mean mean+var"
     n_normalization = 4
-    pca_maf_target = 0.1
+    pca_maf_target = 0
     pca_geno_miss_target = 0.1
 
     // compatibility params
diff --git a/nextflow_schema.json b/nextflow_schema.json
index 174ba51b..afd13c07 100644
--- a/nextflow_schema.json
+++ b/nextflow_schema.json
@@ -168,7 +168,7 @@
         },
         "pca_maf_target": {
           "type": "number",
-          "default": 0.1,
+          "default": 0,
           "description": "Minimum MAF threshold in TARGET samples for variants to be included in the PCA.",
           "minimum": 0,
           "maximum": 1

From cc11c8aa6502ef9c9b6dad8f0365b3a9f5c6fd55 Mon Sep 17 00:00:00 2001
From: Benjamin Wingfield <bwingfield@ebi.ac.uk>
Date: Wed, 31 Jul 2024 11:04:53 +0100
Subject: [PATCH 28/31] add --efo_id parameter

---
 nextflow.config        |  1 +
 nextflow_schema.json   |  6 +++++-
 workflows/pgsc_calc.nf | 11 ++++++++++-
 3 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/nextflow.config b/nextflow.config
index c4abafc8..7824fd44 100644
--- a/nextflow.config
+++ b/nextflow.config
@@ -14,6 +14,7 @@ params {
     scorefile                  = null
     pgs_id                     = null
     trait_efo                  = null
+    efo_id                     = null // synonym for trait_efo, which is deprecated
     pgp_id                     = null
     efo_direct                 = false
 
diff --git a/nextflow_schema.json b/nextflow_schema.json
index 174ba51b..3a445d52 100644
--- a/nextflow_schema.json
+++ b/nextflow_schema.json
@@ -55,10 +55,14 @@
           "description": "A comma separated list of PGS Catalog publications, e.g. PGP000001",
           "pattern": "PGP[0-9]{6}"
         },
-        "trait_efo": {
+        "efo_id": {
           "type": "string",
           "description": "A comma separated list of PGS Catalog EFO traits, e.g. EFO_0004214"
         },
+        "trait_efo": {
+          "type": "string",
+          "description": "DEPRECATED: A comma separated list of PGS Catalog EFO traits, e.g. EFO_0004214"
+        },
         "efo_direct": {
           "type": "boolean",
           "description": "Return only PGS tagged with exact EFO term (e.g. no PGS for child/descendant terms in the ontology)"
diff --git a/workflows/pgsc_calc.nf b/workflows/pgsc_calc.nf
index 8a57e22e..7bded6e6 100644
--- a/workflows/pgsc_calc.nf
+++ b/workflows/pgsc_calc.nf
@@ -170,7 +170,16 @@ workflow PGSCCALC {
     // make sure accessions look sensible before querying PGS Catalog
     def pgs_id = WorkflowPgscCalc.prepareAccessions(params.pgs_id, "pgs_id")
     def pgp_id = WorkflowPgscCalc.prepareAccessions(params.pgp_id, "pgp_id")
-    def trait_efo = WorkflowPgscCalc.prepareAccessions(params.trait_efo, "trait_efo")
+    
+    // temporarily handle parameter synonym (--trait_efo -> --efo_id) 
+    def traits = [params.trait_efo, params.efo_id].findAll { it != null }.join(",")
+    
+    if (params.trait_efo) {
+        println "WARNING: --trait_efo is deprecated and will be removed in a future release, please use --efo_id"
+    }
+
+    def trait_efo = WorkflowPgscCalc.prepareAccessions(traits, "trait_efo")
+    
     def accessions = pgs_id + pgp_id + trait_efo
 
     if (!accessions.every { it.value == "" }) {

From ce4d596929122210a497333842a6707b6ea43ac2 Mon Sep 17 00:00:00 2001
From: Benjamin Wingfield <bwingfield@ebi.ac.uk>
Date: Wed, 31 Jul 2024 11:10:26 +0100
Subject: [PATCH 29/31] update trait_efo -> efo_id

---
 docs/getting-started.rst             | 2 +-
 docs/how-to/calculate_pgscatalog.rst | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/getting-started.rst b/docs/getting-started.rst
index c2c4caf7..5b2dcf49 100644
--- a/docs/getting-started.rst
+++ b/docs/getting-started.rst
@@ -98,7 +98,7 @@ parameter:
     --pgs_id PGS001229 # one score
     --pgs_id PGS001229,PGS001405 # many scores separated by , (no spaces)
 
-.. note:: You can also select scores associated with traits (``--trait_efo``) and
+.. note:: You can also select scores associated with traits (``--efo_id``) and
           publications (``--pgp_id``)
           
 If you would like to use a custom scoring file not published in the PGS Catalog,
diff --git a/docs/how-to/calculate_pgscatalog.rst b/docs/how-to/calculate_pgscatalog.rst
index c04d7877..c7e5929b 100644
--- a/docs/how-to/calculate_pgscatalog.rst
+++ b/docs/how-to/calculate_pgscatalog.rst
@@ -46,12 +46,12 @@ Traits
 ~~~~~~
 
 If you would like to calculate every polygenic score in the Catalog for a
-`trait`_, like `coronary artery disease`_, then you can use the ``--trait_efo``
+`trait`_, like `coronary artery disease`_, then you can use the ``--efo_id``
 parameter:
 
 .. code-block:: console
 
-    --trait_efo EFO_0001645
+    --efi_id EFO_0001645
 
 Multiple traits can be set by using a comma separated list.
 
@@ -85,7 +85,7 @@ Multiple traits can be set by using a comma separated list.
         -profile <docker/singularity/conda> \    
         --input samplesheet.csv \
         --pgs_id PGS001229 \
-        --trait_efo EFO_0001645 \
+        --efo_id EFO_0001645 \
         --pgp_id PGP000001
 
 .. note:: For more details about calculating multiple scores, see :ref:`multiple` 

From 37ad5ed949e5612a5f01fe4c3c96da0202daad6f Mon Sep 17 00:00:00 2001
From: Benjamin Wingfield <bwingfield@ebi.ac.uk>
Date: Wed, 31 Jul 2024 11:12:28 +0100
Subject: [PATCH 30/31] check if --efo_id is set when workflow starts

---
 lib/WorkflowPgscCalc.groovy | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/WorkflowPgscCalc.groovy b/lib/WorkflowPgscCalc.groovy
index 60c4e9fe..d95c68fb 100755
--- a/lib/WorkflowPgscCalc.groovy
+++ b/lib/WorkflowPgscCalc.groovy
@@ -11,8 +11,8 @@ class WorkflowPgscCalc {
     // Check and validate parameters
     //
     public static void initialise(params, log) {
-        if (![params.scorefile, params.pgs_id, params.trait_efo, params.pgp_id].any()) {
-            Nextflow.error " ERROR: You didn't set any scores to use! Please set --scorefile, --pgs_id, --trait_efo, or --pgp_id"
+        if (![params.scorefile, params.pgs_id, params.trait_efo, params.efo_id, params.pgp_id].any()) {
+            Nextflow.error " ERROR: You didn't set any scores to use! Please set --scorefile, --pgs_id, --efo_id, or --pgp_id"
         }
 
         if (!params.target_build) {

From caea73bc69d833b68ef99d0cc2af45fe5cc7e4cf Mon Sep 17 00:00:00 2001
From: Sam Lambert <sam.a.lambert@gmail.com>
Date: Wed, 31 Jul 2024 11:14:05 +0100
Subject: [PATCH 31/31] Typo fix

---
 docs/how-to/calculate_pgscatalog.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/how-to/calculate_pgscatalog.rst b/docs/how-to/calculate_pgscatalog.rst
index c7e5929b..dc2cd55f 100644
--- a/docs/how-to/calculate_pgscatalog.rst
+++ b/docs/how-to/calculate_pgscatalog.rst
@@ -51,7 +51,7 @@ parameter:
 
 .. code-block:: console
 
-    --efi_id EFO_0001645
+    --efo_id EFO_0001645
 
 Multiple traits can be set by using a comma separated list.