From 419417bbd324b32b919a6bef927b13b5100ad8a8 Mon Sep 17 00:00:00 2001 From: waterflow80 Date: Mon, 5 Feb 2024 17:24:03 +0100 Subject: [PATCH 1/2] added a column 'asm_accession' to the sequence_collection_l1 table --- .../ebi/eva/evaseqcol/entities/SeqColLevelOneEntity.java | 7 ++++++- .../ebi/eva/evaseqcol/service/SeqColLevelOneService.java | 8 +++++--- .../uk/ac/ebi/eva/evaseqcol/service/SeqColService.java | 4 ++-- .../controller/AdminControllerIntegrationTest.java | 1 + .../java/uk/ac/ebi/eva/evaseqcol/io/SeqColWriter.java | 6 ++++-- .../eva/evaseqcol/service/SeqColLevelOneServiceTest.java | 8 +++++--- 6 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/main/java/uk/ac/ebi/eva/evaseqcol/entities/SeqColLevelOneEntity.java b/src/main/java/uk/ac/ebi/eva/evaseqcol/entities/SeqColLevelOneEntity.java index dd47949..9a364d7 100644 --- a/src/main/java/uk/ac/ebi/eva/evaseqcol/entities/SeqColLevelOneEntity.java +++ b/src/main/java/uk/ac/ebi/eva/evaseqcol/entities/SeqColLevelOneEntity.java @@ -1,5 +1,6 @@ package uk.ac.ebi.eva.evaseqcol.entities; +import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; import lombok.NoArgsConstructor; import org.hibernate.annotations.Type; @@ -37,10 +38,14 @@ public class SeqColLevelOneEntity extends SeqColEntity{ @Enumerated(EnumType.STRING) protected NamingConvention namingConvention; - public SeqColLevelOneEntity(String digest, NamingConvention namingConvention, JSONLevelOne jsonLevelOne){ + @Column(name = "insdc_accession") + private String asm_accession; // The INSDC assembly accession from which the seqcol was created + + public SeqColLevelOneEntity(String digest, NamingConvention namingConvention, JSONLevelOne jsonLevelOne, String asm_accession){ super(digest, namingConvention); this.seqColLevel1Object = jsonLevelOne; this.namingConvention = namingConvention; + this.asm_accession = asm_accession; } @Override diff --git a/src/main/java/uk/ac/ebi/eva/evaseqcol/service/SeqColLevelOneService.java b/src/main/java/uk/ac/ebi/eva/evaseqcol/service/SeqColLevelOneService.java index 2810409..1e013ea 100644 --- a/src/main/java/uk/ac/ebi/eva/evaseqcol/service/SeqColLevelOneService.java +++ b/src/main/java/uk/ac/ebi/eva/evaseqcol/service/SeqColLevelOneService.java @@ -65,8 +65,9 @@ public List getAllSeqColLevelOneObjects(){ * hold names, lengths and sequences objects*/ public SeqColLevelOneEntity constructSeqColLevelOne(List>> stringListExtendedDataEntities, List>> integerListExtendedDataEntities, - SeqColEntity.NamingConvention convention) throws IOException { + SeqColEntity.NamingConvention convention, String asm_accession) throws IOException { SeqColLevelOneEntity levelOneEntity = new SeqColLevelOneEntity(); + levelOneEntity.setAsm_accession(asm_accession); JSONLevelOne jsonLevelOne = new JSONLevelOne(); // Looping over List types @@ -106,7 +107,8 @@ public SeqColLevelOneEntity constructSeqColLevelOne(List> sequencesExtData = new JSONStringListExtData(levelTwoEntity.getSequences()); JSONExtData> lengthsExtData = new JSONIntegerListExtData(levelTwoEntity.getLengths()); @@ -151,7 +153,7 @@ public SeqColLevelOneEntity constructSeqColLevelOne( lengthsExtEntity ); - return constructSeqColLevelOne(stringListExtendedDataEntities,integerListExtendedDataEntities, convention); + return constructSeqColLevelOne(stringListExtendedDataEntities,integerListExtendedDataEntities, convention, asm_accession); } /** diff --git a/src/main/java/uk/ac/ebi/eva/evaseqcol/service/SeqColService.java b/src/main/java/uk/ac/ebi/eva/evaseqcol/service/SeqColService.java index a733350..5e739a3 100644 --- a/src/main/java/uk/ac/ebi/eva/evaseqcol/service/SeqColService.java +++ b/src/main/java/uk/ac/ebi/eva/evaseqcol/service/SeqColService.java @@ -198,8 +198,8 @@ public IngestionResultEntity fetchAndInsertAllSeqColByAssemblyAccession( // Constructing seqCol Level One object SeqColLevelOneEntity levelOneEntity = levelOneService.constructSeqColLevelOne( - seqColStringListExtDataEntities, seqColIntegerListExtDataEntities, extendedNamesEntity.getNamingConvention() - ); + seqColStringListExtDataEntities, seqColIntegerListExtDataEntities, extendedNamesEntity.getNamingConvention(), + assemblyAccession); try { Optional seqColDigest = insertSeqColL1AndL2( // TODO: Check for possible self invocation problem diff --git a/src/test/java/uk/ac/ebi/eva/evaseqcol/controller/AdminControllerIntegrationTest.java b/src/test/java/uk/ac/ebi/eva/evaseqcol/controller/AdminControllerIntegrationTest.java index 32025fe..49dd5cc 100644 --- a/src/test/java/uk/ac/ebi/eva/evaseqcol/controller/AdminControllerIntegrationTest.java +++ b/src/test/java/uk/ac/ebi/eva/evaseqcol/controller/AdminControllerIntegrationTest.java @@ -106,6 +106,7 @@ void ingestSeqColsTest() { assertTrue(levelTwoEntity.isPresent()); assertEquals(insertedSeqColDigest,levelOneEntity.get().getDigest()); assertNotNull(levelTwoEntity.get().getLengths()); + assertEquals(ASM_ACCESSION, levelOneEntity.get().getAsm_accession()); } @Test diff --git a/src/test/java/uk/ac/ebi/eva/evaseqcol/io/SeqColWriter.java b/src/test/java/uk/ac/ebi/eva/evaseqcol/io/SeqColWriter.java index 39ce40a..c52aad8 100644 --- a/src/test/java/uk/ac/ebi/eva/evaseqcol/io/SeqColWriter.java +++ b/src/test/java/uk/ac/ebi/eva/evaseqcol/io/SeqColWriter.java @@ -146,7 +146,8 @@ public void create() throws IOException { extendedIntegerListDataEntitiesUcsc = (List>>) ucscExtendedDataMap.get("integerListExtDataList"); levelOneEntityUcsc = levelOneService.constructSeqColLevelOne( - extendedStringListDataEntitiesUcsc, extendedIntegerListDataEntitiesUcsc, SeqColEntity.NamingConvention.UCSC); + extendedStringListDataEntitiesUcsc, extendedIntegerListDataEntitiesUcsc, SeqColEntity.NamingConvention.UCSC, + assemblyEntity.getInsdcAccession()); Optional resultDigestUcsc = seqColService.addFullSequenceCollection( levelOneEntityUcsc, extendedStringListDataEntitiesUcsc, extendedIntegerListDataEntitiesUcsc); if (resultDigestUcsc.isPresent()) { @@ -163,7 +164,8 @@ public void create() throws IOException { extendedIntegerListDataEntitiesGenbank = (List>>) genbankExtendedDataMap.get("integerListExtDataList"); levelOneEntityGenbank = levelOneService.constructSeqColLevelOne( - extendedStringListDataEntitiesGenbank, extendedIntegerListDataEntitiesGenbank, SeqColEntity.NamingConvention.GENBANK); + extendedStringListDataEntitiesGenbank, extendedIntegerListDataEntitiesGenbank, SeqColEntity.NamingConvention.GENBANK, + assemblyEntity.getInsdcAccession()); Optional resultDigestGenbank = seqColService.addFullSequenceCollection( levelOneEntityGenbank, extendedStringListDataEntitiesGenbank, extendedIntegerListDataEntitiesGenbank); if (resultDigestGenbank.isPresent()) { diff --git a/src/test/java/uk/ac/ebi/eva/evaseqcol/service/SeqColLevelOneServiceTest.java b/src/test/java/uk/ac/ebi/eva/evaseqcol/service/SeqColLevelOneServiceTest.java index 8d0bd66..eb17b55 100644 --- a/src/test/java/uk/ac/ebi/eva/evaseqcol/service/SeqColLevelOneServiceTest.java +++ b/src/test/java/uk/ac/ebi/eva/evaseqcol/service/SeqColLevelOneServiceTest.java @@ -82,10 +82,12 @@ void constructSeqColL1Test() throws IOException { List>> integerListExtDataList = (List>>) extendedDataMapGenbank.get("integerListExtDataList"); SeqColLevelOneEntity levelOneEntity = levelOneService.constructSeqColLevelOne( - stringListExtDataList, integerListExtDataList, SeqColEntity.NamingConvention.GENBANK); + stringListExtDataList, integerListExtDataList, SeqColEntity.NamingConvention.GENBANK, assemblyEntity.getInsdcAccession()); SeqColLevelTwoEntity levelTwoEntity = levelTwoService. constructSeqColL2(levelOneEntity.getDigest(), stringListExtDataList, integerListExtDataList); - SeqColLevelOneEntity constructedEntity = levelOneService.constructSeqColLevelOne(levelTwoEntity, SeqColEntity.NamingConvention.GENBANK); + SeqColLevelOneEntity constructedEntity = levelOneService.constructSeqColLevelOne(levelTwoEntity, + SeqColEntity.NamingConvention.GENBANK, + assemblyEntity.getInsdcAccession()); assertNotNull(constructedEntity); assertNotNull(constructedEntity.getSeqColLevel1Object().getSequences()); } @@ -100,7 +102,7 @@ void addSequenceCollectionL1() throws IOException { List>> integerListExtDataList = (List>>) extendedDataMapGenbank.get("integerListExtDataList"); SeqColLevelOneEntity levelOneEntity = levelOneService.constructSeqColLevelOne( - stringListExtDataList, integerListExtDataList, SeqColEntity.NamingConvention.GENBANK); + stringListExtDataList, integerListExtDataList, SeqColEntity.NamingConvention.GENBANK, assemblyEntity.getInsdcAccession()); Optional savedEntity = levelOneService.addSequenceCollectionL1(levelOneEntity); assertTrue(savedEntity.isPresent()); System.out.println(savedEntity.get()); From 2c0ee58bab73b774c28be0d3ac141571346a3784 Mon Sep 17 00:00:00 2001 From: waterflow80 Date: Mon, 5 Feb 2024 18:02:34 +0100 Subject: [PATCH 2/2] make the check for existing asm_accession before proceeding with download and the ingestion process --- .../eva/evaseqcol/entities/SeqColLevelOneEntity.java | 7 +++---- .../eva/evaseqcol/repo/SeqColLevelOneRepository.java | 1 + .../eva/evaseqcol/service/SeqColLevelOneService.java | 8 +++++++- .../ac/ebi/eva/evaseqcol/service/SeqColService.java | 11 +++++------ .../controller/AdminControllerIntegrationTest.java | 3 +-- 5 files changed, 17 insertions(+), 13 deletions(-) diff --git a/src/main/java/uk/ac/ebi/eva/evaseqcol/entities/SeqColLevelOneEntity.java b/src/main/java/uk/ac/ebi/eva/evaseqcol/entities/SeqColLevelOneEntity.java index 9a364d7..e91f7ab 100644 --- a/src/main/java/uk/ac/ebi/eva/evaseqcol/entities/SeqColLevelOneEntity.java +++ b/src/main/java/uk/ac/ebi/eva/evaseqcol/entities/SeqColLevelOneEntity.java @@ -1,6 +1,5 @@ package uk.ac.ebi.eva.evaseqcol.entities; -import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; import lombok.NoArgsConstructor; import org.hibernate.annotations.Type; @@ -39,13 +38,13 @@ public class SeqColLevelOneEntity extends SeqColEntity{ protected NamingConvention namingConvention; @Column(name = "insdc_accession") - private String asm_accession; // The INSDC assembly accession from which the seqcol was created + private String asmAccession; // The INSDC assembly accession from which the seqcol was created - public SeqColLevelOneEntity(String digest, NamingConvention namingConvention, JSONLevelOne jsonLevelOne, String asm_accession){ + public SeqColLevelOneEntity(String digest, NamingConvention namingConvention, JSONLevelOne jsonLevelOne, String asmAccession){ super(digest, namingConvention); this.seqColLevel1Object = jsonLevelOne; this.namingConvention = namingConvention; - this.asm_accession = asm_accession; + this.asmAccession = asmAccession; } @Override diff --git a/src/main/java/uk/ac/ebi/eva/evaseqcol/repo/SeqColLevelOneRepository.java b/src/main/java/uk/ac/ebi/eva/evaseqcol/repo/SeqColLevelOneRepository.java index 27ff575..b27bf02 100644 --- a/src/main/java/uk/ac/ebi/eva/evaseqcol/repo/SeqColLevelOneRepository.java +++ b/src/main/java/uk/ac/ebi/eva/evaseqcol/repo/SeqColLevelOneRepository.java @@ -8,6 +8,7 @@ @Repository public interface SeqColLevelOneRepository extends JpaRepository { SeqColLevelOneEntity findSeqColLevelOneEntityByDigest(String digest); + boolean existsByAsmAccession(String asm_accession); long countSeqColLevelOneEntitiesByDigest(String digest); diff --git a/src/main/java/uk/ac/ebi/eva/evaseqcol/service/SeqColLevelOneService.java b/src/main/java/uk/ac/ebi/eva/evaseqcol/service/SeqColLevelOneService.java index 1e013ea..c87f144 100644 --- a/src/main/java/uk/ac/ebi/eva/evaseqcol/service/SeqColLevelOneService.java +++ b/src/main/java/uk/ac/ebi/eva/evaseqcol/service/SeqColLevelOneService.java @@ -45,6 +45,12 @@ public Optional getSeqColLevelOneByDigest(String digest){ } } + /** + * Check whether the given asm_accession exists in the database (in the sequence_collection_level1)*/ + public boolean isAsmAccessionExists(String asm_accession) { + return repository.existsByAsmAccession(asm_accession); + } + public void removeSeqColLevelOneByDigest(String digest) { repository.removeSeqColLevelOneEntityByDigest(digest); } @@ -67,7 +73,7 @@ public SeqColLevelOneEntity constructSeqColLevelOne(List>> integerListExtendedDataEntities, SeqColEntity.NamingConvention convention, String asm_accession) throws IOException { SeqColLevelOneEntity levelOneEntity = new SeqColLevelOneEntity(); - levelOneEntity.setAsm_accession(asm_accession); + levelOneEntity.setAsmAccession(asm_accession); JSONLevelOne jsonLevelOne = new JSONLevelOne(); // Looping over List types diff --git a/src/main/java/uk/ac/ebi/eva/evaseqcol/service/SeqColService.java b/src/main/java/uk/ac/ebi/eva/evaseqcol/service/SeqColService.java index 5e739a3..c6b42c5 100644 --- a/src/main/java/uk/ac/ebi/eva/evaseqcol/service/SeqColService.java +++ b/src/main/java/uk/ac/ebi/eva/evaseqcol/service/SeqColService.java @@ -162,6 +162,10 @@ public void removeAllSeqCol() { public IngestionResultEntity fetchAndInsertAllSeqColByAssemblyAccession( String assemblyAccession) throws IOException, DuplicateSeqColException, AssemblyNotFoundException, AssemblyAlreadyIngestedException{ + if (levelOneService.isAsmAccessionExists(assemblyAccession)) { + logger.warn("Seqcol objects for assembly " + assemblyAccession + " has been already ingested"); + throw new AssemblyAlreadyIngestedException(assemblyAccession); + } Optional> seqColDataMap = ncbiSeqColDataSource .getAllPossibleSeqColExtendedData(assemblyAccession); if (!seqColDataMap.isPresent()) { @@ -218,12 +222,7 @@ public IngestionResultEntity fetchAndInsertAllSeqColByAssemblyAccession( " already exists. Skipping."); } } - if (ingestionResultEntity.getNumberOfInsertedSeqcols() == 0) { - logger.warn("Seqcol objects for assembly " + assemblyAccession + " has been already ingested"); - throw new AssemblyAlreadyIngestedException(assemblyAccession); - } else { - return ingestionResultEntity; - } + return ingestionResultEntity; } diff --git a/src/test/java/uk/ac/ebi/eva/evaseqcol/controller/AdminControllerIntegrationTest.java b/src/test/java/uk/ac/ebi/eva/evaseqcol/controller/AdminControllerIntegrationTest.java index 49dd5cc..83ad425 100644 --- a/src/test/java/uk/ac/ebi/eva/evaseqcol/controller/AdminControllerIntegrationTest.java +++ b/src/test/java/uk/ac/ebi/eva/evaseqcol/controller/AdminControllerIntegrationTest.java @@ -14,7 +14,6 @@ import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.http.client.support.BasicAuthenticationInterceptor; -import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.DynamicPropertyRegistry; import org.springframework.test.context.DynamicPropertySource; import org.springframework.transaction.annotation.Transactional; @@ -106,7 +105,7 @@ void ingestSeqColsTest() { assertTrue(levelTwoEntity.isPresent()); assertEquals(insertedSeqColDigest,levelOneEntity.get().getDigest()); assertNotNull(levelTwoEntity.get().getLengths()); - assertEquals(ASM_ACCESSION, levelOneEntity.get().getAsm_accession()); + assertEquals(ASM_ACCESSION, levelOneEntity.get().getAsmAccession()); } @Test