From f4a577b4c5388dcff1773a353bfe757cba9150f9 Mon Sep 17 00:00:00 2001 From: dbampalikis Date: Wed, 1 Nov 2023 09:18:18 +0100 Subject: [PATCH 01/13] Add endpoint for encrypted files --- api/api.go | 1 + 1 file changed, 1 insertion(+) diff --git a/api/api.go b/api/api.go index caadb6d..b5c831a 100644 --- a/api/api.go +++ b/api/api.go @@ -41,6 +41,7 @@ func Setup() *http.Server { router.GET("/files/:fileid", SelectedMiddleware(), sda.Download) router.GET("/s3/*path", SelectedMiddleware(), s3.Download) router.HEAD("/s3/*path", SelectedMiddleware(), s3.Download) + router.GET("/s3-encrypted/*path", SelectedMiddleware(), s3.Download) router.GET("/health", healthResponse) // Configure TLS settings From 170f9a024449c0a81e4107bcebe67cda0ec7fd06 Mon Sep 17 00:00:00 2001 From: dbampalikis Date: Wed, 1 Nov 2023 09:18:36 +0100 Subject: [PATCH 02/13] Add function for returning encrypted files --- api/s3/s3.go | 55 ++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/api/s3/s3.go b/api/s3/s3.go index 0e333af..947ef13 100644 --- a/api/s3/s3.go +++ b/api/s3/s3.go @@ -188,6 +188,22 @@ func ListObjects(c *gin.Context) { }) } +func getFileInfo(c *gin.Context) (fileInfo *database.FileInfo, err error) { + // Get file info for the given file path (or abort) + fileInfo, err = database.GetDatasetFileInfo(c.Param("dataset"), c.Param("filename")+".c4gh") + if err != nil { + if err.Error() == "sql: no rows in result set" { + c.AbortWithStatus(http.StatusNotFound) + } else { + c.AbortWithStatus(http.StatusInternalServerError) + } + + return + } + + return fileInfo, nil +} + // GetObject respondes to an S3 GetObject request. This request returns S3 // objects. This is done by first fetching any file that matches the dataset + // filename request from the database and then passing the fileID to the @@ -196,15 +212,31 @@ func ListObjects(c *gin.Context) { func GetObject(c *gin.Context) { log.Debugf("S3 GetObject request, context: %v", c.Params) - // Get file info for the given file path (or abort) - fileInfo, err := database.GetDatasetFileInfo(c.Param("dataset"), c.Param("filename")+".c4gh") + fileInfo, err := getFileInfo(c) if err != nil { - if err.Error() == "sql: no rows in result set" { - c.AbortWithStatus(http.StatusNotFound) - } else { - c.AbortWithStatus(http.StatusInternalServerError) - } + return + } + + // Set a param so that Download knows to add S3 headers + c.Set("S3", true) + + // set the fileID so that download knows what file to download + c.Params = append(c.Params, gin.Param{Key: "fileid", Value: fileInfo.FileID}) + // Download the file + sda.Download(c) +} + +// GetEncryptedObject respondes to an S3 GetObject request for encrypted files. +// This request returns S3 objects. This is done by first fetching any file that matches the dataset + +// filename request from the database and then passing the fileID to the +// SDA Download function. +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html +func GetEcnryptedObject(c *gin.Context) { + log.Debugf("S3 GetEncryptedObject request, context: %v", c.Params) + + fileInfo, err := getFileInfo(c) + if err != nil { return } @@ -214,6 +246,9 @@ func GetObject(c *gin.Context) { // set the fileID so that download knows what file to download c.Params = append(c.Params, gin.Param{Key: "fileid", Value: fileInfo.FileID}) + // set the encrypted parameter so that download gets the encrypted file instead + c.Params = append(c.Params, gin.Param{Key: "type", Value: "encrypted"}) + // Download the file sda.Download(c) } @@ -294,7 +329,11 @@ func Download(c *gin.Context) { ListBuckets(c) case c.Param("filename") != "": - GetObject(c) + if strings.Contains(c.Request.URL.String(), "encrypted") { + GetEcnryptedObject(c) + } else { + GetObject(c) + } default: log.Warningf("Got unknown S3 request: %v", c.Request) From e4c37e5a263b5ba7b767acb79f206176ee75d6be Mon Sep 17 00:00:00 2001 From: dbampalikis Date: Wed, 1 Nov 2023 09:19:09 +0100 Subject: [PATCH 03/13] Add case for encrypted files in download --- api/sda/sda.go | 42 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/api/sda/sda.go b/api/sda/sda.go index 614a11e..a47e094 100644 --- a/api/sda/sda.go +++ b/api/sda/sda.go @@ -198,18 +198,48 @@ func Download(c *gin.Context) { return } - // Stitch file and prepare it for streaming - fileStream, err := stitchFile(fileDetails.Header, file, coordinates) - if err != nil { - log.Errorf("could not prepare file for streaming, %s", err) - c.String(http.StatusInternalServerError, "file stream error") + var fileStream io.Reader + switch c.Param("type") { + case "encrypted": + log.Print("Return encrypted file") + fileStream, err = stitchEncryptedFile(fileDetails.Header, file, coordinates) + if err != nil { + log.Errorf("could not prepare file for streaming, %s", err) + c.String(http.StatusInternalServerError, "file stream error") - return + return + } + c.Header("Content-Length", "") + default: + // Stitch file and prepare it for streaming + fileStream, err = stitchFile(fileDetails.Header, file, coordinates) + if err != nil { + log.Errorf("could not prepare file for streaming, %s", err) + c.String(http.StatusInternalServerError, "file stream error") + + return + } } sendStream(c.Writer, fileStream) } +// stitchFile stitches the header and file body together for Crypt4GHReader +// and returns a streamable Reader +var stitchEncryptedFile = func(header []byte, file io.ReadCloser, coordinates *headers.DataEditListHeaderPacket) (io.Reader, error) { + log.Debugf("stitching header to file %s for streaming", file) + // Stitch header and file body together + hr := bytes.NewReader(header) + + encryptedFile := io.MultiReader(hr, io.MultiReader(hr, file)) + + log.Print("Encrypted file:", encryptedFile) + + log.Debugf("file stream for %s constructed", file) + + return encryptedFile, nil +} + // stitchFile stitches the header and file body together for Crypt4GHReader // and returns a streamable Reader var stitchFile = func(header []byte, file io.ReadCloser, coordinates *headers.DataEditListHeaderPacket) (*streaming.Crypt4GHReader, error) { From 089a21e7e27543caf7b23fc33eb65398940b8ea1 Mon Sep 17 00:00:00 2001 From: dbampalikis Date: Wed, 1 Nov 2023 09:54:33 +0100 Subject: [PATCH 04/13] Added biohackrxiv template --- biohackathon/paper.bib | 57 +++++++++++++++++++++++++++++ biohackathon/paper.md | 83 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 140 insertions(+) create mode 100644 biohackathon/paper.bib create mode 100644 biohackathon/paper.md diff --git a/biohackathon/paper.bib b/biohackathon/paper.bib new file mode 100644 index 0000000..9d4188a --- /dev/null +++ b/biohackathon/paper.bib @@ -0,0 +1,57 @@ +@misc{Tryggve1, + title={NeIC {T}ryggve 1}, + url={https://neic.no/tryggve1/}, + author={NeIC}, + publisher={Nordic e-Infrastructure Collaboration}, + year={2014-2017}, + note={Accessed: 2023-03-29} +} + +@misc{NEIC, + title={NeIC}, + url={https://neic.no/}, + author={NeIC}, + publisher={Nordic e-Infrastructure Collaboration}, + note={Accessed: 2023-03-29} +} + +@misc{Tryggve2, + title={NeIC {T}ryggve 2}, + url={https://neic.no/tryggve2/}, + author={NeIC}, + year={2017-2020}, + publisher={Nordic e-Infrastructure Collaboration}, + note={Accessed: 2023-03-29} +} + +@misc{Heilsa, + title={NeIC {H}eilsa {T}ryggvedottir}, + url={https://neic.no/heilsa/}, + author={NeIC}, + year={2021-2024}, + publisher={Nordic e-Infrastructure Collaboration}, + note={Accessed: 2023-03-29} +} + +@article{EGA, + Title={The European Genome-phenome Archive in 2021}, + Author={Freeberg, Mallory Ann and Fromont, Lauren A and D'Altri, Teresa and Romero, Anna Foix and Ciges, Jorge Izquierdo and Jene, Aina and Kerry, Giselle and Moldes, Mauricio and Ariosa, Roberto and Bahena, Silvia and Barrowdale, Daniel and Barbero, Marcos Casado and Fernandez-Orth, Dietmar and Garcia-Linares, Carles and Garcia-Rios, Emilio and Haziza, Frédéric and Juhasz, Bela and Llobet, Oscar Martinez and Milla, Gemma and Mohan, Anand and Rueda, Manuel and Sankar, Aravind and Shaju, Dona and Shimpi, Ashutosh and Singh, Babita and Thomas, Coline and de la Torre, Sabela and Uyan, Umuthan and Vasallo, Claudia and Flicek, Paul and Guigo, Roderic and Navarro, Arcadi and Parkinson, Helen and Keane, Thomas and Rambla, Jordi}, + DOI={10.1093/nar/gkab1059}, + Number={D1}, + Volume={50}, + Month={January}, + Year={2022}, + Journal={Nucleic acids research}, + ISSN={0305-1048}, + Pages={D980—D987}, + Abstract={The European Genome-phenome Archive (EGA - https://ega-archive.org/) is a resource for long term secure archiving of all types of potentially identifiable genetic, phenotypic, and clinical data resulting from biomedical research projects. Its mission is to foster hosted data reuse, enable reproducibility, and accelerate biomedical and translational research in line with the FAIR principles. Launched in 2008, the EGA has grown quickly, currently archiving over 4,500 studies from nearly one thousand institutions. The EGA operates a distributed data access model in which requests are made to the data controller, not to the EGA, therefore, the submitter keeps control on who has access to the data and under which conditions. Given the size and value of data hosted, the EGA is constantly improving its value chain, that is, how the EGA can contribute to enhancing the value of human health data by facilitating its submission, discovery, access, and distribution, as well as leading the design and implementation of standards and methods necessary to deliver the value chain. The EGA has become a key GA4GH Driver Project, leading multiple development efforts and implementing new standards and tools, and has been appointed as an ELIXIR Core Data Resource.}, + URL={https://europepmc.org/articles/PMC8728218}, +} + +@misc{FEGA, + title={Federated EGA}, + url={https://ega-archive.org/federated}, + author={EGA\\ Consortium}, + publisher={EGA}, + note={Accessed: 2023-03-29} +} \ No newline at end of file diff --git a/biohackathon/paper.md b/biohackathon/paper.md new file mode 100644 index 0000000..c8dacdb --- /dev/null +++ b/biohackathon/paper.md @@ -0,0 +1,83 @@ +--- +title: 'BioHackrXiv template' +tags: + - replace with your own keywords + - at least 3 is recommended +authors: + - name: First Last + orcid: 0000-0000-0000-0000 + affiliation: 1 + - name: Second Last + orcid: 0000-0000-0000-0000 + affiliation: 2 + +affiliations: + - name: Institution 1, address, city, country + index: 1 + - name: Institution 1, address, city, country + index: 2 +date: 01 January 2020 +bibliography: paper.bib +authors_short: Last et al. (2021) BioHackrXiv template +group: BioHackrXiv +event: BioHackathon Europe 2021 +--- + +# Introduction or Background + +Add to this section a couple of paragraphs introducing the work done dring the BioHackathon, CodeFest, VoCamp or Sprint event. Please add references whenever needed, for instance [@Katayama:2010]. + +Please separate paragraphs with a double line. + +## Subsection level 2 + +Please keep sections to a maximum of three levels, even better if only two levels. + +### Subsection level 3 + +Please keep sections to a maximum of three levels. + +## Tables, figures and so on + +Please remember to introduce tables (see Table 1) before they appear on the document. We recommend to center tables, formulas and figure but not the corresponding captions. Feel free to modify the table style as it better suits to your data. + +Table 1 +| Header 1 | Header 2 | +| -------- | -------- | +| item 1 | item 2 | +| item 3 | item 4 | + +Remember to introduce figures (see Figure 1) before they appear on the document. + +![BioHackrXiv logo](./biohackrxiv.png) + +Figure 1. A figure corresponding to the logo of our BioHackrXiv preprint. + +# Other main section on your manuscript level 1 + +Feel free to use numbered lists or bullet points as you need. +* Item 1 +* Item 2 + +# Discussion and/or Conclusion + +We recommend to include some discussion or conclusion about your work. Feel free to modify the section title as it fits better to your manuscript. + +# Future work + +And maybe you want to add a sentence or two on how you plan to continue. Please keep reading to learn about citations and references. + +For citations of references, we prefer the use of parenthesis, last name and year. If you use a citation manager, Elsevier – Harvard or American Psychological Association (APA) will work. If you are referencing web pages, software or so, please do so in the same way. Whenever possible, add authors and year. We have included a couple of citations along this document for you to get the idea. Please remember to always add DOI whenever available, if not possible, please provide alternative URLs. You will end up with an alphabetical order list by authors’ last name. + +# Jupyter notebooks, GitHub repositories and data repositories + +* Please add a list here +* Make sure you let us know which of these correspond to Jupyter notebooks. Although not supported yet, we plan to add features for them +* And remember, software and data need a license for them to be used by others, no license means no clear rules so nobody could legally use a non-licensed research object, whatever that object is + +# Acknowledgements +Please always remember to acknowledge the BioHackathon, CodeFest, VoCamp, Sprint or similar where this work was (partially) developed. + +# References + +Leave thise section blank, create a paper.bib with all your references. From c2c3332437646ecb54426968765a8b38dddabd46 Mon Sep 17 00:00:00 2001 From: dbampalikis Date: Wed, 1 Nov 2023 10:29:47 +0100 Subject: [PATCH 05/13] Added authors for the biohackathon paper --- biohackathon/paper.md | 41 +++++++++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/biohackathon/paper.md b/biohackathon/paper.md index c8dacdb..7faff6b 100644 --- a/biohackathon/paper.md +++ b/biohackathon/paper.md @@ -4,18 +4,43 @@ tags: - replace with your own keywords - at least 3 is recommended authors: - - name: First Last - orcid: 0000-0000-0000-0000 + - name: Johan Viklund + orcid: 0000-0003-1984-8522 + affiliation: 3 + - name: Stefan Negru + orcid: 0000-0002-6544-5022 affiliation: 1 - - name: Second Last - orcid: 0000-0000-0000-0000 + - name: Dimitris Bampalikis + affiliation: 3 + orcid: 0000-0002-2078-3079 + - name: Liisa Lado-Villar + orcid: + affiliation: 1 + - name: Alexandros Dimopoulos + orcid: 0000-0002-4602-2040 affiliation: 2 + - name: Konstantinos Koumpouras + orcid: 0000-0001-5598-4703 + affiliation: 3 + - name: Alexandros Aperis + orcid: 0000-0002-1699-2476 + affiliation: 3 + - name: Panagiotis Chatzopoulos + orcid: 0009-0004-7445-2453 + affiliation: 3 + - name: Marko Malenic + orcid: 0009-0007-3824-8449 + affiliation: 4 affiliations: - - name: Institution 1, address, city, country - index: 1 - - name: Institution 1, address, city, country - index: 2 + - name: CSC – IT CENTER FOR SCIENCE, Espoo, Finland + index: 1 + - name: Biomedical Sciences Research Center Alexander Fleming, Vari, Greece + index: 2 + - name: National Bioinformatics Infrastructure Sweden (NBIS), Uppsala University, SciLifeLab, ICM - Department of Cell and Molecular Biology, Uppsala, Sweden. + index: 3 + - name: University of Melbourne, Melbourne, AU + index: 4 date: 01 January 2020 bibliography: paper.bib authors_short: Last et al. (2021) BioHackrXiv template From e49dac69f374c4b11cdf87949b02a90975acaa67 Mon Sep 17 00:00:00 2001 From: Panos Chatzopoulos Date: Wed, 1 Nov 2023 15:15:11 +0100 Subject: [PATCH 06/13] add bam header endpoint. Co-authored-by: Kostas Koumpouras Co-authored-by: Panos Chatzopoulos --- api/api.go | 1 + api/s3/s3.go | 3 +++ api/sda/sda.go | 17 +++++++++++++++++ go.mod | 1 + go.sum | 8 ++++++++ 5 files changed, 30 insertions(+) diff --git a/api/api.go b/api/api.go index b5c831a..bcb7035 100644 --- a/api/api.go +++ b/api/api.go @@ -43,6 +43,7 @@ func Setup() *http.Server { router.HEAD("/s3/*path", SelectedMiddleware(), s3.Download) router.GET("/s3-encrypted/*path", SelectedMiddleware(), s3.Download) router.GET("/health", healthResponse) + router.GET("/header/*path", SelectedMiddleware(), s3.Download) // Configure TLS settings log.Info("(3/5) Configuring TLS") diff --git a/api/s3/s3.go b/api/s3/s3.go index 947ef13..52ce024 100644 --- a/api/s3/s3.go +++ b/api/s3/s3.go @@ -222,6 +222,9 @@ func GetObject(c *gin.Context) { // set the fileID so that download knows what file to download c.Params = append(c.Params, gin.Param{Key: "fileid", Value: fileInfo.FileID}) + if strings.Contains(c.Request.URL.String(), "header") { + c.Params = append(c.Params, gin.Param{Key: "type", Value: "header"}) + } // Download the file sda.Download(c) diff --git a/api/sda/sda.go b/api/sda/sda.go index a47e094..f5ac6f7 100644 --- a/api/sda/sda.go +++ b/api/sda/sda.go @@ -11,6 +11,7 @@ import ( "strings" "time" + "github.com/biogo/hts/bam" "github.com/gin-gonic/gin" "github.com/neicnordic/crypt4gh/model/headers" "github.com/neicnordic/crypt4gh/streaming" @@ -220,7 +221,23 @@ func Download(c *gin.Context) { return } } + if c.Param("type") == "header" { + bamReader, err := bam.NewReader(fileStream, 0) + if err != nil { + log.Fatalln(err) + } + defer bamReader.Close() + reader := bamReader + h := reader.Header() + var r io.Reader + buf := new(bytes.Buffer) + fmt.Fprint(buf, h) + r = buf + sendStream(c.Writer, r) + return + + } sendStream(c.Writer, fileStream) } diff --git a/go.mod b/go.mod index b95e914..9e25c2a 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.20 require ( github.com/DATA-DOG/go-sqlmock v1.5.0 github.com/aws/aws-sdk-go v1.46.2 + github.com/biogo/hts v1.4.4 github.com/dgraph-io/ristretto v0.1.1 github.com/gin-gonic/gin v1.9.1 github.com/google/uuid v1.3.1 diff --git a/go.sum b/go.sum index ecb590c..3176bbf 100644 --- a/go.sum +++ b/go.sum @@ -45,6 +45,10 @@ github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q github.com/aws/aws-sdk-go v1.33.0/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go v1.46.2 h1:XZbOmjtN1VCfEtQq7QNFsbxIqO+bB+bRhiOBjp6AzWc= github.com/aws/aws-sdk-go v1.46.2/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/biogo/boom v0.0.0-20150317015657-28119bc1ffc1 h1:LAHY5JxqhOgJDeDBGKsQ4300qd3sG8C0j5CQS8gD+Kw= +github.com/biogo/boom v0.0.0-20150317015657-28119bc1ffc1/go.mod h1:fwtxkutinkQcME9Zlywh66T0jZLLjgrwSLY2WxH2N3U= +github.com/biogo/hts v1.4.4 h1:Z+TminqAKRE/t6nyy5PwI/DL90kdew4GpghB+QdjjFk= +github.com/biogo/hts v1.4.4/go.mod h1:AfPn4uJQ2zxi04Q/4vccdmCX16W+IsHXVguPsdh4HE4= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM= github.com/bytedance/sonic v1.10.0 h1:qtNZduETEIWJVIyDl01BeNxur2rW9OwTQ/yBqFRkKEk= @@ -194,8 +198,10 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02 github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/kortschak/utter v0.0.0-20190412033250-50fe362e6560/go.mod h1:oDr41C7kH9wvAikWyFhr6UFr8R7nelpmCF5XR5rL7I8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -284,6 +290,7 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -629,6 +636,7 @@ google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= From 6883b26ac60527257d4867ae8fb9626392faf9cf Mon Sep 17 00:00:00 2001 From: Panos Chatzopoulos Date: Wed, 1 Nov 2023 17:21:11 +0100 Subject: [PATCH 07/13] add endpoint for c4gh header size Co-authored-by: Kostas Koumpouras Co-authored-by: Panos Chatzopoulos " --- api/api.go | 1 + api/s3/s3.go | 2 ++ api/sda/sda.go | 15 +++++++++++++++ 3 files changed, 18 insertions(+) diff --git a/api/api.go b/api/api.go index bcb7035..44dc34d 100644 --- a/api/api.go +++ b/api/api.go @@ -44,6 +44,7 @@ func Setup() *http.Server { router.GET("/s3-encrypted/*path", SelectedMiddleware(), s3.Download) router.GET("/health", healthResponse) router.GET("/header/*path", SelectedMiddleware(), s3.Download) + router.GET("/c4gheadsize/*path", SelectedMiddleware(), s3.Download) // Configure TLS settings log.Info("(3/5) Configuring TLS") diff --git a/api/s3/s3.go b/api/s3/s3.go index 52ce024..f215197 100644 --- a/api/s3/s3.go +++ b/api/s3/s3.go @@ -224,6 +224,8 @@ func GetObject(c *gin.Context) { c.Params = append(c.Params, gin.Param{Key: "fileid", Value: fileInfo.FileID}) if strings.Contains(c.Request.URL.String(), "header") { c.Params = append(c.Params, gin.Param{Key: "type", Value: "header"}) + } else if strings.Contains(c.Request.URL.String(), "c4gheadsize") { + c.Params = append(c.Params, gin.Param{Key: "type", Value: "headersize"}) } // Download the file diff --git a/api/sda/sda.go b/api/sda/sda.go index f5ac6f7..2f2df02 100644 --- a/api/sda/sda.go +++ b/api/sda/sda.go @@ -221,6 +221,7 @@ func Download(c *gin.Context) { return } } + log.Debug("- - - - - - - test 1 - - - - - - - ") if c.Param("type") == "header" { bamReader, err := bam.NewReader(fileStream, 0) if err != nil { @@ -237,7 +238,21 @@ func Download(c *gin.Context) { return + } else if c.Param("type") == "headersize" { + log.Debug("- - - - - - - test 2 - - - - - - - ") + head := fileDetails.Header + log.Debug("head: ", head) + //headlength := len(head) + headlength := bytes.NewReader(head) + log.Debug("headlength size: ", headlength.Size()) + buf := new(bytes.Buffer) + len := strconv.Itoa(int(headlength.Size())) + fmt.Fprint(buf, len) + sendStream(c.Writer, buf) + + return } + log.Debug("- - - - - - - test 3 - - - - - - - ") sendStream(c.Writer, fileStream) } From ac9b84aef1a6f7260bc1df83c4bbc00aa1444ca1 Mon Sep 17 00:00:00 2001 From: Liisa Lado-Villar <47525503+lilachic@users.noreply.github.com> Date: Thu, 2 Nov 2023 11:46:08 +0100 Subject: [PATCH 08/13] modify dev_utils README to contain M1 chip help and reordering --- dev_utils/README.md | 44 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/dev_utils/README.md b/dev_utils/README.md index 146ffe3..db90a17 100644 --- a/dev_utils/README.md +++ b/dev_utils/README.md @@ -1,7 +1,18 @@ # Local testing howto ## Getting started locally -In root repository +The commands are run in root repository `cd sda-download`. +First create the necessary credentials in dev_utils directory. + +```command +cd dev_utils +sh make_certs.sh +``` + + +Return to app root +`cd ../` + ``` export CONFIGFILE="./dev_utils/config.yaml" @@ -10,11 +21,8 @@ go run cmd/main.go This requires having the certificates generated and the database up. -First create the necessary credentials. - -```command -sh make_certs.sh -``` +> [!NOTE] +> Download does not start if you have not run integrations tests, for there are no keys in place. ## Getting up and running fast with docker compose @@ -28,6 +36,30 @@ For testing the API sh run_integration_test_no_tls.sh ``` +> [!NOTE] +> For Mac M1/M2 users the `dev_utils/compose.yml` file needs some modificatation to be used with ARM processor architecture. All the containers need to be build for `platform: "linux/arm64"` for running the Linux VM on top of M1/M2 ARM laptop. Also the used binaries and base images need to be compiled/build for ARM. As an example the part for S3 container is below, do note the selected version being build as multiplatform container version and added port exposing change. + +```command +s3: + platform: "linux/arm64" + command: server /data --console-address ":9001" + container_name: s3 + environment: + - MINIO_ACCESS_KEY=access + - MINIO_SECRET_KEY=secretkey + - MINIO_SERVER_URL=https://127.0.0.1:9000 + healthcheck: + test: ["CMD", "curl", "-fkq", "https://localhost:9000/minio/health/live"] + interval: 5s + timeout: 20s + retries: 3 + image: minio/minio:RELEASE.2023-10-25T06-33-25Z + ports: + - "9000:9000" + - "9001:9001" +... +``` + ## Starting the services using docker compose with TLS enabled To start all the backend services using docker compose. From 0533d2e454d44bb94829d14f91f635f834c92da9 Mon Sep 17 00:00:00 2001 From: dbampalikis Date: Thu, 2 Nov 2023 14:43:21 +0100 Subject: [PATCH 09/13] (WIP) Added the first sections of the biohackathon papaer --- biohackathon/paper.bib | 55 ++++++++++++++++++++++++++++++++++++------ biohackathon/paper.md | 11 ++++++--- 2 files changed, 56 insertions(+), 10 deletions(-) diff --git a/biohackathon/paper.bib b/biohackathon/paper.bib index 9d4188a..a6e033c 100644 --- a/biohackathon/paper.bib +++ b/biohackathon/paper.bib @@ -48,10 +48,51 @@ @article{EGA URL={https://europepmc.org/articles/PMC8728218}, } -@misc{FEGA, - title={Federated EGA}, - url={https://ega-archive.org/federated}, - author={EGA\\ Consortium}, - publisher={EGA}, - note={Accessed: 2023-03-29} -} \ No newline at end of file +@misc{GDI, + title={Genomic Data Infrastructure}, + url={https://gdi.onemilliongenomes.eu/}, + author={GDI\\ Consortium}, + publisher={GDI}, + note={Accessed: 2023-11-02} +} + +@article{crypt4gh, + author = {Senf, Alexander and Davies, Robert and Haziza, Frédéric and Marshall, John and Troncoso-Pastoriza, Juan and Hofmann, Oliver and Keane, Thomas M.}, + title = "{Crypt4GH: a file format standard enabling native access to encrypted data}", + journal = {Bioinformatics}, + volume = {37}, + number = {17}, + pages = {2753-2754}, + year = {2021}, + month = {02}, + abstract = "{The majority of genome analysis tools and pipelines require data to be decrypted for access. This potentially leaves sensitive genetic data exposed, either because the unencrypted data is not removed after analysis, or because the data leaves traces on the permanent storage medium.: We defined a file container specification enabling direct byte-level compatible random access to encrypted genetic data stored in community standards such as SAM/BAM/CRAM/VCF/BCF. By standardizing this format, we show how it can be added as a native file format to genomic libraries, enabling direct analysis of encrypted data without the need to create a decrypted copy.The Crypt4GH specification can be found at: http://samtools.github.io/hts-specs/crypt4gh.pdf.Supplementary data are available at Bioinformatics online.}", + issn = {1367-4803}, + doi = {10.1093/bioinformatics/btab087}, + url = {https://doi.org/10.1093/bioinformatics/btab087}, + eprint = {https://academic.oup.com/bioinformatics/article-pdf/37/17/2753/50339113/btab087.pdf}, +} + +@article{htsget, + author = {Kelleher, Jerome and Lin, Mike and Albach, C H and Birney, Ewan and Davies, Robert and Gourtovaia, Marina and Glazer, David and Gonzalez, Cristina Y and Jackson, David K and Kemp, Aaron and Marshall, John and Nowak, Andrew and Senf, Alexander and Tovar-Corona, Jaime M and Vikhorev, Alexander and Keane, Thomas M and GA4GH Streaming Task Team }, + title = "{htsget: a protocol for securely streaming genomic data}", + journal = {Bioinformatics}, + volume = {35}, + number = {1}, + pages = {119-121}, + year = {2018}, + month = {06}, + abstract = "{Standardized interfaces for efficiently accessing high-throughput sequencing data are a fundamental requirement for large-scale genomic data sharing. We have developed htsget, a protocol for secure, efficient and reliable access to sequencing read and variation data. We demonstrate four independent client and server implementations, and the results of a comprehensive interoperability demonstration.http://samtools.github.io/hts-specs/htsget.htmlSupplementary data are available at Bioinformatics online.}", + issn = {1367-4803}, + doi = {10.1093/bioinformatics/bty492}, + url = {https://doi.org/10.1093/bioinformatics/bty492}, + eprint = {https://academic.oup.com/bioinformatics/article-pdf/35/1/119/48962810/bioinformatics\_35\_1\_119.pdf}, +} + + +@misc{htsget-rs, + title={htsget rust implementation}, + url={https://github.com/umccr/htsget-rs}, + author={htsget-rs team}, + publisher={htsget-rs team}, + note={Accessed: 2023-11-02} +} diff --git a/biohackathon/paper.md b/biohackathon/paper.md index 7faff6b..a4f5992 100644 --- a/biohackathon/paper.md +++ b/biohackathon/paper.md @@ -50,13 +50,18 @@ event: BioHackathon Europe 2021 # Introduction or Background -Add to this section a couple of paragraphs introducing the work done dring the BioHackathon, CodeFest, VoCamp or Sprint event. Please add references whenever needed, for instance [@Katayama:2010]. +The European Genome-phenome Archive (EGA) [@EGA] is a service for archiving and sharing personally identifiable genetic and phenotypic data, while the The Genomic Data Infrastructure (GDI) [@GDI] project is enabling access to genomic and related phenotypic and clinical data across Europe. Both projects are focused on creating federated and secure infrastructure for researchers to archive and share data with the research community, to support further research. -Please separate paragraphs with a double line. + +The project was focused on the data access part of the infrastructure. The files are encrypted in the archives, using the crypt4gh standard [@crypt4gh]. Currently, there exist data access processes, where the files are either decrypted on the server side and then transferred to the user or re-encrypted server-side and provided to the user in an outbox. + + +Htsget [@htsget] as a data access protocol also allows access to parts of files. Before the Biohackathon event, there did not exist any production-level client tools that supports access to encrypted data. The main goal of the project was to create a client tool that can access encrypted data over the htsget protocol, able to work with the GA4GH Passport and Visa standard, which enhances the security of the data access interfaces. ## Subsection level 2 -Please keep sections to a maximum of three levels, even better if only two levels. +In order to enable for random data access on encrypted files, we worked on the htsget-rs [@htsget-rs], a Rust htsget server to support the aforementioned standards and the sda-download, an implementation handling the data-out API of the archives, developed by the Nordic collaboration under the umbrella of the Nordic e-Infrastructure Collaboration(NeIC) [@NEIC]. + ### Subsection level 3 From c1288ba7af53a218257ffe2dc9716cdf11e1d800 Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Thu, 2 Nov 2023 15:22:42 +0100 Subject: [PATCH 10/13] Updates to the paper --- biohackathon/paper.md | 38 ++++++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/biohackathon/paper.md b/biohackathon/paper.md index a4f5992..937276b 100644 --- a/biohackathon/paper.md +++ b/biohackathon/paper.md @@ -50,17 +50,47 @@ event: BioHackathon Europe 2021 # Introduction or Background -The European Genome-phenome Archive (EGA) [@EGA] is a service for archiving and sharing personally identifiable genetic and phenotypic data, while the The Genomic Data Infrastructure (GDI) [@GDI] project is enabling access to genomic and related phenotypic and clinical data across Europe. Both projects are focused on creating federated and secure infrastructure for researchers to archive and share data with the research community, to support further research. +The European Genome-phenome Archive (EGA) [@EGA] and it's extension the +Federated EGA (FEGA) (@FEGA) are services for archiving and sharing personally +identifiable genetic and phenotypic data, while The Genomic Data Infrastructure +(GDI) [@GDI] project is enabling secondary use of genomic and phenotypic +clinical data across Europe. Both projects are focused on creating federated +and secure infrastructure for researchers to archive and share data with the +research community, to support further research. -The project was focused on the data access part of the infrastructure. The files are encrypted in the archives, using the crypt4gh standard [@crypt4gh]. Currently, there exist data access processes, where the files are either decrypted on the server side and then transferred to the user or re-encrypted server-side and provided to the user in an outbox. +This project was focused on the data access part of the infrastructure. The +files are encrypted in the archives, using the crypt4gh standard [@crypt4gh]. +Currently, we have a data access processes, where the files are either +decrypted on the server side and then transferred to the user or re-encrypted +server-side with the users public key and provided to the user in a dedicated +outbox. This process is cumbersome and requires a lot of manual intervention by +the archive operators. It's also not very granular, for example, in the case a +user only wants access to just a small region of the genomic files we still +provide the full reencrypted dataset, which is expensive in both human and +computational resources. -Htsget [@htsget] as a data access protocol also allows access to parts of files. Before the Biohackathon event, there did not exist any production-level client tools that supports access to encrypted data. The main goal of the project was to create a client tool that can access encrypted data over the htsget protocol, able to work with the GA4GH Passport and Visa standard, which enhances the security of the data access interfaces. +Htsget [@htsget] as a data access protocol allows access to parts of files. +Before the Biohackathon event, there were no htsget servers that supported +partial access to encrypted data. Our goal of the project was to extend the +htsget-rs [@htsget-rs] server and integrate it into the GDI starter kit to +support GA4GH Passport authorized, re-encrypted access to partial files. + + +BLA BLA BLA CLIENT +create a client tool that can access encrypted data over the htsget protocol, +able to work with the GA4GH Passport and Visa standard, which enhances the +security of the data access interfaces. + ## Subsection level 2 -In order to enable for random data access on encrypted files, we worked on the htsget-rs [@htsget-rs], a Rust htsget server to support the aforementioned standards and the sda-download, an implementation handling the data-out API of the archives, developed by the Nordic collaboration under the umbrella of the Nordic e-Infrastructure Collaboration(NeIC) [@NEIC]. +In order to enable for random data access on encrypted files, we worked on the +htsget-rs [@htsget-rs], a Rust htsget server to support the aforementioned +standards and the sda-download, an implementation handling the data-out API of +the archives, developed by the Nordic collaboration under the umbrella of the +Nordic e-Infrastructure Collaboration(NeIC) [@NEIC]. ### Subsection level 3 From 996e6fa7851695edbcf1fe1014e832f28daa26cd Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Thu, 2 Nov 2023 15:27:01 +0100 Subject: [PATCH 11/13] Add sequence diagram. Co-Authored-By: Stefan Negru --- biohackathon/paper.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/biohackathon/paper.md b/biohackathon/paper.md index 937276b..ee5fadf 100644 --- a/biohackathon/paper.md +++ b/biohackathon/paper.md @@ -93,6 +93,23 @@ the archives, developed by the Nordic collaboration under the umbrella of the Nordic e-Infrastructure Collaboration(NeIC) [@NEIC]. +```mermaid +sequenceDiagram + Htsget Client->>Htsget Server: GET reference=chr1&start=0&end=200 + activate Htsget Server + Htsget Server->>Htsget Client: URL tickets + deactivate Htsget Server + opt Get Htsget File info + Htsget Server->>Download API: HEAD (unencrypted file size) + activate Download API + Htsget Server->>Download API: GET (file index) + Htsget Server->>Download API: GET (underlying file header) + Htsget Server->>Download API: GET (crypt4gh header size) + deactivate Download API + end + Htsget Client->>Download API: GET File (HTTP HEADER Client-Public-Key) +``` + ### Subsection level 3 Please keep sections to a maximum of three levels. From 0b1cc82c9607d9057df3b243cf75622e932b13c4 Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Fri, 3 Nov 2023 10:25:18 +0100 Subject: [PATCH 12/13] More paper stuff --- biohackathon/paper.md | 100 ++++++++++++++++++++++++++---------------- 1 file changed, 62 insertions(+), 38 deletions(-) diff --git a/biohackathon/paper.md b/biohackathon/paper.md index ee5fadf..630550d 100644 --- a/biohackathon/paper.md +++ b/biohackathon/paper.md @@ -48,7 +48,7 @@ group: BioHackrXiv event: BioHackathon Europe 2021 --- -# Introduction or Background +# Introduction The European Genome-phenome Archive (EGA) [@EGA] and it's extension the Federated EGA (FEGA) (@FEGA) are services for archiving and sharing personally @@ -56,7 +56,11 @@ identifiable genetic and phenotypic data, while The Genomic Data Infrastructure (GDI) [@GDI] project is enabling secondary use of genomic and phenotypic clinical data across Europe. Both projects are focused on creating federated and secure infrastructure for researchers to archive and share data with the -research community, to support further research. +research community, to support further research. In the nordics we collaborate +on a sofware suite under the NeIC umbrella called the Sensitive Data Archive +[@ref] to support these efforts. + +(work it in Nordic e-Infrastructure Collaboration(NeIC) [@NEIC].) This project was focused on the data access part of the infrastructure. The @@ -74,23 +78,27 @@ computational resources. Htsget [@htsget] as a data access protocol allows access to parts of files. Before the Biohackathon event, there were no htsget servers that supported partial access to encrypted data. Our goal of the project was to extend the -htsget-rs [@htsget-rs] server and integrate it into the GDI starter kit to +htsget-rs [@htsget-rs] Rust server and integrate it into the GDI starter kit to support GA4GH Passport authorized, re-encrypted access to partial files. -BLA BLA BLA CLIENT -create a client tool that can access encrypted data over the htsget protocol, -able to work with the GA4GH Passport and Visa standard, which enhances the -security of the data access interfaces. +We also aimed to extend already existing client tools so they can access +encrypted data over the htsget protocol using GA4GH Passport and Visa standard, +which enhances the security of the data access interfaces. + + +# Results + +## HTSGet -## Subsection level 2 -In order to enable for random data access on encrypted files, we worked on the -htsget-rs [@htsget-rs], a Rust htsget server to support the aforementioned -standards and the sda-download, an implementation handling the data-out API of -the archives, developed by the Nordic collaboration under the umbrella of the -Nordic e-Infrastructure Collaboration(NeIC) [@NEIC]. +In order to enable for random data access on encrypted files, we worked on +extending htsget-rs [@htsget-rs] to work with the sda-download part of the NeIC +nordic sensitive data archiving suite. We developed the following sequence +diagram for interactions between the server softwares. Basically it means the +client first communicates with the Rust server to get information on what to +download, which it then downloads from the sda-download service. ```mermaid @@ -110,49 +118,65 @@ sequenceDiagram Htsget Client->>Download API: GET File (HTTP HEADER Client-Public-Key) ``` -### Subsection level 3 -Please keep sections to a maximum of three levels. +## Extend sda-download to support re-encryption -## Tables, figures and so on -Please remember to introduce tables (see Table 1) before they appear on the document. We recommend to center tables, formulas and figure but not the corresponding captions. Feel free to modify the table style as it better suits to your data. +We also extended the sda-download service to support re-encryption of +requested files. This so the user can get files encrypted with their own +keypair instead of just plain unencrypted files. Since we don't want to keep +the archive secret key in a service that is available directly from the +internet we have implemented a small microservice (gRPC Server in the diagram) +that recieves the encrypted header and a public key and re-encrypts and sends +it back. -Table 1 -| Header 1 | Header 2 | -| -------- | -------- | -| item 1 | item 2 | -| item 3 | item 4 | -Remember to introduce figures (see Figure 1) before they appear on the document. + ```mermaid +sequenceDiagram + client->>sda-download: HTTP Header: Crypt4gh-Public-Key + activate sda-download + sda-download->>gRPC Server: Request: OldHeader, PublicKey + activate gRPC Server + gRPC Server->>sda-download: NewHeader + deactivate gRPC Server + sda-download->>client: encrypted Data + deactivate sda-download + ``` -![BioHackrXiv logo](./biohackrxiv.png) - -Figure 1. A figure corresponding to the logo of our BioHackrXiv preprint. -# Other main section on your manuscript level 1 +## Bits and bobs -Feel free to use numbered lists or bullet points as you need. -* Item 1 -* Item 2 -# Discussion and/or Conclusion + * Updated the htsget starter-kit to use the htsget-rs + * Enhanced sda-cli with a htsget command + * Drank lots of coffee + * Found and fixed a bunch of bugs. + * Started implementing native support for crypt4gh in htsget-rs -We recommend to include some discussion or conclusion about your work. Feel free to modify the section title as it fits better to your manuscript. # Future work -And maybe you want to add a sentence or two on how you plan to continue. Please keep reading to learn about citations and references. -For citations of references, we prefer the use of parenthesis, last name and year. If you use a citation manager, Elsevier – Harvard or American Psychological Association (APA) will work. If you are referencing web pages, software or so, please do so in the same way. Whenever possible, add authors and year. We have included a couple of citations along this document for you to get the idea. Please remember to always add DOI whenever available, if not possible, please provide alternative URLs. You will end up with an alphabetical order list by authors’ last name. + * Fully implement crypt4gh in htsget-rs + * Enable support for crypt4gh edit lists + * Authenticating requests/tickets, so the download service can trust that a + request came from the htsget-rs service + * Remove unnecesarry bytes + * Showcase reading an encrypted file over the htsget protocol. + + +# GitHub repositories -# Jupyter notebooks, GitHub repositories and data repositories +* [GenomicDataInfrastructure/starter-kit-storage-and-interfaces](https://github.com/GenomicDataInfrastructure/starter-kit-storage-and-interfaces) +* [GenomicDataInfrastructure/starter-kit-htsget](https://github.com/GenomicDataInfrastructure/starer-kit-htsget) +* [umccr/htsget-rs](https://github.com/umccr/htsget-rs) +* [neicnordic/sda-download](https://github.com/neicnordic/sda-download) +* [neicnordic/sensitive-data-archive](https://github.com/neicnordic/sensitive-data-archive) +* [NBISweden/sda-cli](https://github.com/NBISweden/sda-cli) -* Please add a list here -* Make sure you let us know which of these correspond to Jupyter notebooks. Although not supported yet, we plan to add features for them -* And remember, software and data need a license for them to be used by others, no license means no clear rules so nobody could legally use a non-licensed research object, whatever that object is # Acknowledgements +TODO Copy from last paper Please always remember to acknowledge the BioHackathon, CodeFest, VoCamp, Sprint or similar where this work was (partially) developed. # References From fbc45ed5885524032c5b0e0dac859c17a7f1af85 Mon Sep 17 00:00:00 2001 From: Stefan Negru Date: Tue, 28 Nov 2023 10:40:43 +0200 Subject: [PATCH 13/13] line from download api to htsget-client --- biohackathon/paper.md | 1 + 1 file changed, 1 insertion(+) diff --git a/biohackathon/paper.md b/biohackathon/paper.md index 630550d..8e204f6 100644 --- a/biohackathon/paper.md +++ b/biohackathon/paper.md @@ -116,6 +116,7 @@ sequenceDiagram deactivate Download API end Htsget Client->>Download API: GET File (HTTP HEADER Client-Public-Key) + Download API->>Htsget Client: re-encrypted file ```