From ff40725e4e0adf39e2ef23922fc8e99b814d2adf Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Thu, 7 Nov 2024 09:03:17 -0600 Subject: [PATCH 1/7] save at second structural change --- .../curation_api/dao/CrossReferenceDAO.java | 21 +++ .../jobs/executors/BiogridOrcExecutor.java | 124 +++++++++++++++--- .../services/BioGridOrcsService.java | 52 ++++++++ .../services/CrossReferenceService.java | 5 + .../dto/fms/BioGridOrcsFmsDTOValidator.java | 32 +++++ 5 files changed, 213 insertions(+), 21 deletions(-) create mode 100644 src/main/java/org/alliancegenome/curation_api/services/BioGridOrcsService.java create mode 100644 src/main/java/org/alliancegenome/curation_api/services/validation/dto/fms/BioGridOrcsFmsDTOValidator.java diff --git a/src/main/java/org/alliancegenome/curation_api/dao/CrossReferenceDAO.java b/src/main/java/org/alliancegenome/curation_api/dao/CrossReferenceDAO.java index 162d95391..a7fda812a 100644 --- a/src/main/java/org/alliancegenome/curation_api/dao/CrossReferenceDAO.java +++ b/src/main/java/org/alliancegenome/curation_api/dao/CrossReferenceDAO.java @@ -4,11 +4,16 @@ import jakarta.persistence.Query; import org.alliancegenome.curation_api.dao.base.BaseSQLDAO; import org.alliancegenome.curation_api.model.entities.CrossReference; +import org.alliancegenome.curation_api.model.entities.Gene; import org.alliancegenome.curation_api.model.entities.ResourceDescriptorPage; +import io.quarkus.logging.Log; + +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; @ApplicationScoped public class CrossReferenceDAO extends BaseSQLDAO { @@ -32,6 +37,22 @@ public Map getGenesWithCrossRefs(ResourceDescriptorPage page) { return ensemblGeneMap; } + public Map getGenesWithCrossRefs(Set referencedCuries) { + String sql = """ + select gc.genomicentity_id, cr.referencedcurie from genomicentity_crossreference as gc, crossreference as cr + where gc.crossreferences_id = cr.id AND cr.referencedCurie IN (:referencedCuries) + """; + Query query = entityManager.createNativeQuery(sql); + query.setParameter("referencedCuries", referencedCuries); + List objects = query.getResultList(); + Map idCurieMap = new HashMap<>(); + objects.forEach(object -> { + idCurieMap.put((String) object[1], (Long) object[0]); + }); + return idCurieMap; + + } + public Integer persistAccessionGeneAssociated(Long crossReferenceID, Long geneID) { String sql = """ insert into genomicentity_crossreference (crossreferences_id,genomicentity_id) diff --git a/src/main/java/org/alliancegenome/curation_api/jobs/executors/BiogridOrcExecutor.java b/src/main/java/org/alliancegenome/curation_api/jobs/executors/BiogridOrcExecutor.java index b90f3f015..df6d3fb4f 100644 --- a/src/main/java/org/alliancegenome/curation_api/jobs/executors/BiogridOrcExecutor.java +++ b/src/main/java/org/alliancegenome/curation_api/jobs/executors/BiogridOrcExecutor.java @@ -1,17 +1,28 @@ package org.alliancegenome.curation_api.jobs.executors; import java.io.FileInputStream; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.zip.GZIPInputStream; +import org.alliancegenome.curation_api.dao.CrossReferenceDAO; +import org.alliancegenome.curation_api.dao.GeneDAO; +import org.alliancegenome.curation_api.dao.ResourceDescriptorPageDAO; import org.alliancegenome.curation_api.exceptions.ObjectUpdateException.ObjectUpdateExceptionData; import org.alliancegenome.curation_api.jobs.util.CsvSchemaBuilder; +import org.alliancegenome.curation_api.model.entities.CrossReference; +import org.alliancegenome.curation_api.model.entities.Gene; +import org.alliancegenome.curation_api.model.entities.ResourceDescriptorPage; import org.alliancegenome.curation_api.model.entities.bulkloads.BulkLoadFileHistory; import org.alliancegenome.curation_api.model.ingest.dto.fms.BiogridOrcFmsDTO; +import org.alliancegenome.curation_api.response.SearchResponse; +import org.alliancegenome.curation_api.services.CrossReferenceService; +import org.alliancegenome.curation_api.services.DataProviderService; import org.alliancegenome.curation_api.util.ProcessDisplayHelper; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; @@ -22,19 +33,50 @@ import com.fasterxml.jackson.dataformat.csv.CsvParser; import com.fasterxml.jackson.dataformat.csv.CsvSchema; +import io.quarkus.logging.Log; import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import jakarta.transaction.Transactional; @ApplicationScoped public class BiogridOrcExecutor extends LoadFileExecutor { + @Inject + ResourceDescriptorPageDAO resourceDescriptorPageDAO; + + @Inject + GeneDAO geneDAO; + + @Inject + CrossReferenceDAO crossRefDAO; + + @Inject + CrossReferenceService crossReferenceService; + + //Todo: remove this and add to the service method once it's created + @Transactional public void execLoad(BulkLoadFileHistory bulkLoadFileHistory) { try (TarArchiveInputStream tarInputStream = new TarArchiveInputStream( - new GZIPInputStream(new FileInputStream(bulkLoadFileHistory.getBulkLoadFile().getLocalFilePath())))) { + new GZIPInputStream(new FileInputStream(bulkLoadFileHistory.getBulkLoadFile().getLocalFilePath())))) { TarArchiveEntry tarEntry; Set biogridIds = new HashSet<>(); + HashMap rdpParams = new HashMap<>(); + rdpParams.put("name", "biogrid/orcs"); + ResourceDescriptorPage resourceDescriptorPage = resourceDescriptorPageDAO.findByParams(rdpParams) + .getSingleResult(); + + int index = 0; + while ((tarEntry = tarInputStream.getNextEntry()) != null) { + if (tarEntry.getName().equals("BIOGRID-ORCS-SCREEN_1558-1.1.16.screen.tab.txt")) { + Log.debug("----------Starting file: -----------------------"); + Log.debug(tarEntry.getName()); + } + Log.debug("----------------on loop number:----------"); + Log.debug(index); + index++; CsvMapper csvMapper = new CsvMapper(); CsvSchema biogridOrcFmsSchema = CsvSchemaBuilder.biogridOrcFmsSchema(); @@ -54,7 +96,7 @@ public void execLoad(BulkLoadFileHistory bulkLoadFileHistory) { .readValues(tarInputStream.readAllBytes()); List biogridData = it.readAll(); - runLoad(bulkLoadFileHistory, biogridData, biogridIds); + runLoad(bulkLoadFileHistory, biogridData, resourceDescriptorPage); } } catch (Exception e) { @@ -63,32 +105,50 @@ public void execLoad(BulkLoadFileHistory bulkLoadFileHistory) { } } - private boolean runLoad(BulkLoadFileHistory history, List biogridList, Set biogridIds) { + private boolean runLoad(BulkLoadFileHistory history, List biogridList, ResourceDescriptorPage resourceDescriptorPage) { ProcessDisplayHelper ph = new ProcessDisplayHelper(); ph.addDisplayHandler(loadProcessDisplayService); if (CollectionUtils.isNotEmpty(biogridList)) { - String loadMessage = biogridList.get(0).getClass().getSimpleName() + " update"; - ph.startProcess(loadMessage, biogridList.size()); + try { + String loadMessage = "BioGrid update"; + Set referencedCuries = populateEntrezIdsFromFiles(biogridList, history); + ph.startProcess(loadMessage, referencedCuries.size()); + updateHistory(history); + + Map genomicEntityCrossRefMap = crossReferenceService.getGenomicEntityCrossRefMap(referencedCuries); + + for (String referencedCurie : genomicEntityCrossRefMap.keySet()) { + + HashMap crossRefParams = new HashMap<>(); + crossRefParams.put("referencedCurie", referencedCurie); + crossRefParams.put("displayName", referencedCurie); + crossRefParams.put("resourceDescriptorPage.id", resourceDescriptorPage.getId()); + + SearchResponse crossRefDupSearch = + crossRefDAO.findByParams(crossRefParams); + + if(!crossRefDupSearch.getResults().isEmpty()) continue; + + CrossReference newCrossRef = new CrossReference(); + newCrossRef.setReferencedCurie(referencedCurie); + newCrossRef.setDisplayName("BioGRID CRISPR Screen Cell Line Phenotypes"); + newCrossRef.setResourceDescriptorPage(resourceDescriptorPage); + + crossRefDAO.persist(newCrossRef); + + crossRefDAO.persistAccessionGeneAssociated(newCrossRef.getId(), genomicEntityCrossRefMap.get(referencedCurie)); + + + history.incrementCompleted(); - updateHistory(history); - for (BiogridOrcFmsDTO biogridOrcFmsDTO : biogridList) { - try { - if (biogridOrcFmsDTO.getIdentifierType().equals("ENTREZ_GENE")) { - String identifier = "NCBI_Gene:" + biogridOrcFmsDTO.getIdentifierId(); - biogridIds.add(identifier); - history.incrementCompleted(); - } else { - history.incrementSkipped(); - - } - } catch (Exception e) { - e.printStackTrace(); - history.incrementFailed(); - addException(history, - new ObjectUpdateExceptionData(biogridOrcFmsDTO, e.getMessage(), e.getStackTrace())); } + } catch (Exception e) { + e.printStackTrace(); + history.incrementFailed(); + addException(history, new ObjectUpdateExceptionData(biogridOrcFmsDTO, e.getMessage(), e.getStackTrace())); ph.progressProcess(); } + history.incrementCompleted(); updateHistory(history); updateExceptions(history); ph.finishProcess(); @@ -96,4 +156,26 @@ private boolean runLoad(BulkLoadFileHistory history, List biog return true; } + + private Set populateEntrezIdsFromFiles(List biogridList, BulkLoadFileHistory history) { + Set biogridIds = new HashSet<>(); + + for (BiogridOrcFmsDTO biogridOrcFmsDTO : biogridList) { + try { + if (!biogridOrcFmsDTO.getIdentifierType().equals("ENTREZ_GENE")){ + history.incrementSkipped(); + continue; + } + + String identifier = "NCBI_Gene:" + biogridOrcFmsDTO.getIdentifierId(); + biogridIds.add(identifier); + + } catch (Exception e) { + e.printStackTrace(); + } + } + + return biogridIds; + + } } \ No newline at end of file diff --git a/src/main/java/org/alliancegenome/curation_api/services/BioGridOrcsService.java b/src/main/java/org/alliancegenome/curation_api/services/BioGridOrcsService.java new file mode 100644 index 000000000..4c401f886 --- /dev/null +++ b/src/main/java/org/alliancegenome/curation_api/services/BioGridOrcsService.java @@ -0,0 +1,52 @@ +package org.alliancegenome.curation_api.services; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import org.alliancegenome.curation_api.constants.EntityFieldConstants; +import org.alliancegenome.curation_api.dao.CrossReferenceDAO; +import org.alliancegenome.curation_api.dao.SequenceTargetingReagentDAO; +import org.alliancegenome.curation_api.enums.BackendBulkDataProvider; +import org.alliancegenome.curation_api.exceptions.ValidationException; +import org.alliancegenome.curation_api.interfaces.crud.BaseUpsertServiceInterface; +import org.alliancegenome.curation_api.model.entities.CrossReference; +import org.alliancegenome.curation_api.model.entities.SequenceTargetingReagent; +import org.alliancegenome.curation_api.model.ingest.dto.fms.SequenceTargetingReagentFmsDTO; +import org.alliancegenome.curation_api.services.base.SubmittedObjectCrudService; +import org.alliancegenome.curation_api.services.validation.dto.fms.BioGridOrcsFmsDTOValidator; +import org.alliancegenome.curation_api.services.validation.dto.fms.SequenceTargetingReagentFmsDTOValidator; + +import jakarta.annotation.PostConstruct; +import jakarta.enterprise.context.RequestScoped; +import jakarta.inject.Inject; +import jakarta.transaction.Transactional; + +@RequestScoped +public class BioGridOrcsService { + + @Inject BioGridOrcsFmsDTOValidator bioGridOrcsFmsDTOValidator; + @Inject CrossReferenceDAO crossReferenceDAO; + + // @Override + // @PostConstruct + // protected void init() { + // setSQLDao(crossReferenceDAO); + // } + + @Transactional + public CrossReference insert(BioGridOrcsFmsDTOValidator dto, BackendBulkDataProvider dataProvider) throws ValidationException { + // CrossReference crossReference = bioGridOrcsFmsDTOValidator.validateSQTRFmsDTO(dto, dataProvider); + return new CrossReference(); + // return crossReferenceDAO.persist(sqtr); + } + + // public List getIdsByDataProvider(String dataProvider) { + // Map params = new HashMap<>(); + // params.put(EntityFieldConstants.DATA_PROVIDER, dataProvider); + // List ids = sqtrDAO.findIdsByParams(params); + // ids.removeIf(Objects::isNull); + // return ids; + // } +} diff --git a/src/main/java/org/alliancegenome/curation_api/services/CrossReferenceService.java b/src/main/java/org/alliancegenome/curation_api/services/CrossReferenceService.java index eff5ba8d7..87a1644bc 100644 --- a/src/main/java/org/alliancegenome/curation_api/services/CrossReferenceService.java +++ b/src/main/java/org/alliancegenome/curation_api/services/CrossReferenceService.java @@ -16,6 +16,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; @RequestScoped public class CrossReferenceService extends BaseEntityCrudService { @@ -109,4 +110,8 @@ public String getCrossReferenceUniqueId(CrossReference xref) { public Map getGenomicEntityCrossRefMap(ResourceDescriptorPage page) { return crossReferenceDAO.getGenesWithCrossRefs(page); } + + public Map getGenomicEntityCrossRefMap(Set referencedCuries) { + return crossReferenceDAO.getGenesWithCrossRefs(referencedCuries); + } } diff --git a/src/main/java/org/alliancegenome/curation_api/services/validation/dto/fms/BioGridOrcsFmsDTOValidator.java b/src/main/java/org/alliancegenome/curation_api/services/validation/dto/fms/BioGridOrcsFmsDTOValidator.java new file mode 100644 index 000000000..92c6e9a2e --- /dev/null +++ b/src/main/java/org/alliancegenome/curation_api/services/validation/dto/fms/BioGridOrcsFmsDTOValidator.java @@ -0,0 +1,32 @@ +package org.alliancegenome.curation_api.services.validation.dto.fms; + +import java.util.HashMap; + +import org.alliancegenome.curation_api.enums.BackendBulkDataProvider; +import org.alliancegenome.curation_api.model.entities.CrossReference; +import org.alliancegenome.curation_api.model.ingest.dto.fms.BiogridOrcFmsDTO; +import org.alliancegenome.curation_api.response.SearchResponse; + +import jakarta.enterprise.context.RequestScoped; + +@RequestScoped +public class BioGridOrcsFmsDTOValidator { + + //Todo: rename? + public CrossReference validateBioGridOrcsFmsDTO(String referencedCurie){ + // HashMap crossRefParams = new HashMap<>(); + // crossRefParams.put("referencedCurie", referencedCurie); + // crossRefParams.put("displayName", referencedCurie); + // crossRefParams.put("resourceDescriptorPage.id", resourceDescriptorPage.getId()); + + // // Log.debug("--------------crossRefDupSearch----------------"); + // SearchResponse crossRefDupSearch = + // crossRefDAO.findByParams(crossRefParams); + // Log.debug(crossRefDupSearch.getResults().isEmpty()); + + // if(!crossRefDupSearch.getResults().isEmpty()) continue; + + return new CrossReference(); + } + +} From 8dd803b768f536f9ecf0703e398226baf3b82185 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Tue, 12 Nov 2024 15:23:56 -0600 Subject: [PATCH 2/7] SCRUM-4513 add cleanup code --- .../curation_api/dao/CrossReferenceDAO.java | 4 - .../jobs/executors/BiogridOrcExecutor.java | 94 +++++++++++-------- .../services/DataProviderService.java | 15 +++ 3 files changed, 69 insertions(+), 44 deletions(-) diff --git a/src/main/java/org/alliancegenome/curation_api/dao/CrossReferenceDAO.java b/src/main/java/org/alliancegenome/curation_api/dao/CrossReferenceDAO.java index a7fda812a..02724376d 100644 --- a/src/main/java/org/alliancegenome/curation_api/dao/CrossReferenceDAO.java +++ b/src/main/java/org/alliancegenome/curation_api/dao/CrossReferenceDAO.java @@ -4,12 +4,8 @@ import jakarta.persistence.Query; import org.alliancegenome.curation_api.dao.base.BaseSQLDAO; import org.alliancegenome.curation_api.model.entities.CrossReference; -import org.alliancegenome.curation_api.model.entities.Gene; import org.alliancegenome.curation_api.model.entities.ResourceDescriptorPage; -import io.quarkus.logging.Log; - -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; diff --git a/src/main/java/org/alliancegenome/curation_api/jobs/executors/BiogridOrcExecutor.java b/src/main/java/org/alliancegenome/curation_api/jobs/executors/BiogridOrcExecutor.java index df6d3fb4f..2d5f8fa6a 100644 --- a/src/main/java/org/alliancegenome/curation_api/jobs/executors/BiogridOrcExecutor.java +++ b/src/main/java/org/alliancegenome/curation_api/jobs/executors/BiogridOrcExecutor.java @@ -1,10 +1,12 @@ package org.alliancegenome.curation_api.jobs.executors; import java.io.FileInputStream; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -13,16 +15,16 @@ import org.alliancegenome.curation_api.dao.CrossReferenceDAO; import org.alliancegenome.curation_api.dao.GeneDAO; import org.alliancegenome.curation_api.dao.ResourceDescriptorPageDAO; -import org.alliancegenome.curation_api.exceptions.ObjectUpdateException.ObjectUpdateExceptionData; import org.alliancegenome.curation_api.jobs.util.CsvSchemaBuilder; import org.alliancegenome.curation_api.model.entities.CrossReference; -import org.alliancegenome.curation_api.model.entities.Gene; +import org.alliancegenome.curation_api.model.entities.DataProvider; +import org.alliancegenome.curation_api.model.entities.Organization; import org.alliancegenome.curation_api.model.entities.ResourceDescriptorPage; import org.alliancegenome.curation_api.model.entities.bulkloads.BulkLoadFileHistory; import org.alliancegenome.curation_api.model.ingest.dto.fms.BiogridOrcFmsDTO; -import org.alliancegenome.curation_api.response.SearchResponse; import org.alliancegenome.curation_api.services.CrossReferenceService; import org.alliancegenome.curation_api.services.DataProviderService; +import org.alliancegenome.curation_api.services.OrganizationService; import org.alliancegenome.curation_api.util.ProcessDisplayHelper; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; @@ -33,10 +35,8 @@ import com.fasterxml.jackson.dataformat.csv.CsvParser; import com.fasterxml.jackson.dataformat.csv.CsvSchema; -import io.quarkus.logging.Log; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; -import jakarta.transaction.Transactional; @ApplicationScoped public class BiogridOrcExecutor extends LoadFileExecutor { @@ -53,30 +53,40 @@ public class BiogridOrcExecutor extends LoadFileExecutor { @Inject CrossReferenceService crossReferenceService; - //Todo: remove this and add to the service method once it's created - @Transactional + @Inject + OrganizationService organizationService; + + @Inject + DataProviderService dataProviderService; + public void execLoad(BulkLoadFileHistory bulkLoadFileHistory) { try (TarArchiveInputStream tarInputStream = new TarArchiveInputStream( new GZIPInputStream(new FileInputStream(bulkLoadFileHistory.getBulkLoadFile().getLocalFilePath())))) { TarArchiveEntry tarEntry; - Set biogridIds = new HashSet<>(); + List biogridData = new ArrayList<>(); + String name = bulkLoadFileHistory.getBulkLoad().getName(); + String dataProviderName = name.substring(0, name.indexOf(" ")); + + Organization organization = organizationService.getByAbbr(dataProviderName).getEntity(); HashMap rdpParams = new HashMap<>(); rdpParams.put("name", "biogrid/orcs"); ResourceDescriptorPage resourceDescriptorPage = resourceDescriptorPageDAO.findByParams(rdpParams) .getSingleResult(); - int index = 0; + List dataProviderIdsBefore = new ArrayList<>( + dataProviderService.getDataProviderMap(organization, resourceDescriptorPage) + .values() + .stream() + .map(DataProvider::getId) + .toList()); + + dataProviderIdsBefore.removeIf(Objects::isNull); + + List dataProviderIdsLoaded = new ArrayList<>(); while ((tarEntry = tarInputStream.getNextEntry()) != null) { - if (tarEntry.getName().equals("BIOGRID-ORCS-SCREEN_1558-1.1.16.screen.tab.txt")) { - Log.debug("----------Starting file: -----------------------"); - Log.debug(tarEntry.getName()); - } - Log.debug("----------------on loop number:----------"); - Log.debug(index); - index++; CsvMapper csvMapper = new CsvMapper(); CsvSchema biogridOrcFmsSchema = CsvSchemaBuilder.biogridOrcFmsSchema(); @@ -95,17 +105,23 @@ public void execLoad(BulkLoadFileHistory bulkLoadFileHistory) { .with(biogridOrcFmsSchema) .readValues(tarInputStream.readAllBytes()); - List biogridData = it.readAll(); - runLoad(bulkLoadFileHistory, biogridData, resourceDescriptorPage); + biogridData.addAll(it.readAll()); } + runLoad(bulkLoadFileHistory, biogridData, resourceDescriptorPage, organization, dataProviderService, + dataProviderIdsLoaded); + + runCleanup(dataProviderService, bulkLoadFileHistory, dataProviderName, dataProviderIdsBefore, + dataProviderIdsLoaded, "Biogrid Orc Load Type"); } catch (Exception e) { failLoad(bulkLoadFileHistory, e); e.printStackTrace(); } } - private boolean runLoad(BulkLoadFileHistory history, List biogridList, ResourceDescriptorPage resourceDescriptorPage) { + private boolean runLoad(BulkLoadFileHistory history, List biogridList, + ResourceDescriptorPage resourceDescriptorPage, Organization organization, + DataProviderService dataProviderService, List dataProviderIdsLoaded) { ProcessDisplayHelper ph = new ProcessDisplayHelper(); ph.addDisplayHandler(loadProcessDisplayService); if (CollectionUtils.isNotEmpty(biogridList)) { @@ -115,37 +131,35 @@ private boolean runLoad(BulkLoadFileHistory history, List biog ph.startProcess(loadMessage, referencedCuries.size()); updateHistory(history); - Map genomicEntityCrossRefMap = crossReferenceService.getGenomicEntityCrossRefMap(referencedCuries); - - for (String referencedCurie : genomicEntityCrossRefMap.keySet()) { - - HashMap crossRefParams = new HashMap<>(); - crossRefParams.put("referencedCurie", referencedCurie); - crossRefParams.put("displayName", referencedCurie); - crossRefParams.put("resourceDescriptorPage.id", resourceDescriptorPage.getId()); - - SearchResponse crossRefDupSearch = - crossRefDAO.findByParams(crossRefParams); + Map genomicEntityCrossRefMap = crossReferenceService + .getGenomicEntityCrossRefMap(referencedCuries); - if(!crossRefDupSearch.getResults().isEmpty()) continue; + for (String referencedCurie : referencedCuries) { CrossReference newCrossRef = new CrossReference(); newCrossRef.setReferencedCurie(referencedCurie); newCrossRef.setDisplayName("BioGRID CRISPR Screen Cell Line Phenotypes"); newCrossRef.setResourceDescriptorPage(resourceDescriptorPage); - - crossRefDAO.persist(newCrossRef); - - crossRefDAO.persistAccessionGeneAssociated(newCrossRef.getId(), genomicEntityCrossRefMap.get(referencedCurie)); - - - history.incrementCompleted(); + + DataProvider provider = new DataProvider(); + provider.setSourceOrganization(organization); + provider.setCrossReference(newCrossRef); + + DataProvider entity = dataProviderService + .insertBioGridOrcDataProvider(provider, genomicEntityCrossRefMap.get(referencedCurie)) + .getEntity(); + + if (entity != null) { + dataProviderIdsLoaded.add(entity.getId()); + history.incrementCompleted(); + } else { + history.incrementSkipped(); + } } } catch (Exception e) { e.printStackTrace(); history.incrementFailed(); - addException(history, new ObjectUpdateExceptionData(biogridOrcFmsDTO, e.getMessage(), e.getStackTrace())); ph.progressProcess(); } history.incrementCompleted(); @@ -162,7 +176,7 @@ private Set populateEntrezIdsFromFiles(List biogridLis for (BiogridOrcFmsDTO biogridOrcFmsDTO : biogridList) { try { - if (!biogridOrcFmsDTO.getIdentifierType().equals("ENTREZ_GENE")){ + if (!biogridOrcFmsDTO.getIdentifierType().equals("ENTREZ_GENE")) { history.incrementSkipped(); continue; } diff --git a/src/main/java/org/alliancegenome/curation_api/services/DataProviderService.java b/src/main/java/org/alliancegenome/curation_api/services/DataProviderService.java index f19f2e68f..7feebaa4f 100644 --- a/src/main/java/org/alliancegenome/curation_api/services/DataProviderService.java +++ b/src/main/java/org/alliancegenome/curation_api/services/DataProviderService.java @@ -92,6 +92,21 @@ public ObjectResponse insertExpressionAtlasDataProvider(DataProvid return new ObjectResponse<>(dbEntity); } + @Transactional + public ObjectResponse insertBioGridOrcDataProvider(DataProvider entity, Long geneticEntityId) { + String referencedCurie = entity.getCrossReference().getReferencedCurie(); + + DataProvider dbEntity = getDataProvider(entity.getSourceOrganization(), referencedCurie, entity.getCrossReference().getResourceDescriptorPage()); + + // we only create new records, no updates + if (dbEntity == null) { + dataProviderDAO.persist(entity); + crossReferenceDAO.persistAccessionGeneAssociated(entity.getCrossReference().getId(), geneticEntityId); + return new ObjectResponse<>(entity); + } + return new ObjectResponse<>(dbEntity); + } + @NotNull public static String getFullReferencedCurie(String localReferencedCurie) { return RESOURCE_DESCRIPTOR_PREFIX + ":" + localReferencedCurie; From 09778ae7b4b7773ae9f288138ff750d8ef82f456 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Wed, 13 Nov 2024 14:53:55 -0600 Subject: [PATCH 3/7] SCRUM-4513 began adding integration tests --- .../crud/CrossReferenceCrudController.java | 11 ++++ .../crud/CrossReferenceCrudInterface.java | 12 ++++ .../jobs/executors/BiogridOrcExecutor.java | 23 ++++++- .../ingest/dto/fms/BiogridOrcFmsDTO.java | 4 -- .../dto/fms/BiogridOrcIngestFmsDTO.java | 16 +++++ .../services/BioGridOrcsService.java | 52 ---------------- .../dto/fms/BioGridOrcsFmsDTOValidator.java | 32 ---------- .../BiogridOrcBulkUploadFmsITCase.java | 61 +++++++++++++++++++ .../bulk/fms/12_biogrid/AF_01_all_fields.json | 32 ++++++++++ 9 files changed, 152 insertions(+), 91 deletions(-) create mode 100644 src/main/java/org/alliancegenome/curation_api/model/ingest/dto/fms/BiogridOrcIngestFmsDTO.java delete mode 100644 src/main/java/org/alliancegenome/curation_api/services/BioGridOrcsService.java delete mode 100644 src/main/java/org/alliancegenome/curation_api/services/validation/dto/fms/BioGridOrcsFmsDTOValidator.java create mode 100644 src/test/java/org/alliancegenome/curation_api/BiogridOrcBulkUploadFmsITCase.java create mode 100644 src/test/resources/bulk/fms/12_biogrid/AF_01_all_fields.json diff --git a/src/main/java/org/alliancegenome/curation_api/controllers/crud/CrossReferenceCrudController.java b/src/main/java/org/alliancegenome/curation_api/controllers/crud/CrossReferenceCrudController.java index 48de548e8..c9704a1e6 100644 --- a/src/main/java/org/alliancegenome/curation_api/controllers/crud/CrossReferenceCrudController.java +++ b/src/main/java/org/alliancegenome/curation_api/controllers/crud/CrossReferenceCrudController.java @@ -3,7 +3,10 @@ import org.alliancegenome.curation_api.controllers.base.BaseEntityCrudController; import org.alliancegenome.curation_api.dao.CrossReferenceDAO; import org.alliancegenome.curation_api.interfaces.crud.CrossReferenceCrudInterface; +import org.alliancegenome.curation_api.jobs.executors.BiogridOrcExecutor; import org.alliancegenome.curation_api.model.entities.CrossReference; +import org.alliancegenome.curation_api.model.ingest.dto.fms.BiogridOrcIngestFmsDTO; +import org.alliancegenome.curation_api.response.APIResponse; import org.alliancegenome.curation_api.services.CrossReferenceService; import jakarta.annotation.PostConstruct; @@ -16,9 +19,17 @@ public class CrossReferenceCrudController extends BaseEntityCrudController { + + @POST + @Path("/bulk/{dataProvider}/biogridfile") + @JsonView(View.FieldsAndLists.class) + APIResponse updateBiogridOrc(@PathParam("dataProvider") String dataProvider, BiogridOrcIngestFmsDTO biogridOrcData); } \ No newline at end of file diff --git a/src/main/java/org/alliancegenome/curation_api/jobs/executors/BiogridOrcExecutor.java b/src/main/java/org/alliancegenome/curation_api/jobs/executors/BiogridOrcExecutor.java index 2d5f8fa6a..c431187de 100644 --- a/src/main/java/org/alliancegenome/curation_api/jobs/executors/BiogridOrcExecutor.java +++ b/src/main/java/org/alliancegenome/curation_api/jobs/executors/BiogridOrcExecutor.java @@ -22,6 +22,8 @@ import org.alliancegenome.curation_api.model.entities.ResourceDescriptorPage; import org.alliancegenome.curation_api.model.entities.bulkloads.BulkLoadFileHistory; import org.alliancegenome.curation_api.model.ingest.dto.fms.BiogridOrcFmsDTO; +import org.alliancegenome.curation_api.response.APIResponse; +import org.alliancegenome.curation_api.response.LoadHistoryResponce; import org.alliancegenome.curation_api.services.CrossReferenceService; import org.alliancegenome.curation_api.services.DataProviderService; import org.alliancegenome.curation_api.services.OrganizationService; @@ -72,8 +74,7 @@ public void execLoad(BulkLoadFileHistory bulkLoadFileHistory) { HashMap rdpParams = new HashMap<>(); rdpParams.put("name", "biogrid/orcs"); - ResourceDescriptorPage resourceDescriptorPage = resourceDescriptorPageDAO.findByParams(rdpParams) - .getSingleResult(); + ResourceDescriptorPage resourceDescriptorPage = resourceDescriptorPageDAO.findByParams(rdpParams).getSingleResult(); List dataProviderIdsBefore = new ArrayList<>( dataProviderService.getDataProviderMap(organization, resourceDescriptorPage) @@ -140,7 +141,7 @@ private boolean runLoad(BulkLoadFileHistory history, List biog newCrossRef.setReferencedCurie(referencedCurie); newCrossRef.setDisplayName("BioGRID CRISPR Screen Cell Line Phenotypes"); newCrossRef.setResourceDescriptorPage(resourceDescriptorPage); - + DataProvider provider = new DataProvider(); provider.setSourceOrganization(organization); provider.setCrossReference(newCrossRef); @@ -171,6 +172,22 @@ private boolean runLoad(BulkLoadFileHistory history, List biog return true; } + public APIResponse runLoadApi(String dataProviderName, List biogridDTOs) { + List dataProviderIdsLoaded = new ArrayList<>(); + Organization organization = organizationService.getByAbbr(dataProviderName).getEntity(); + + HashMap rdpParams = new HashMap<>(); + rdpParams.put("name", "biogrid/orcs"); + ResourceDescriptorPage resourceDescriptorPage = resourceDescriptorPageDAO.findByParams(rdpParams).getSingleResult(); + + BulkLoadFileHistory history = new BulkLoadFileHistory(biogridDTOs.size()); + history = bulkLoadFileHistoryDAO.persist(history); + runLoad(history, biogridDTOs, resourceDescriptorPage, organization, dataProviderService, dataProviderIdsLoaded); + history.finishLoad(); + + return new LoadHistoryResponce(history); + } + private Set populateEntrezIdsFromFiles(List biogridList, BulkLoadFileHistory history) { Set biogridIds = new HashSet<>(); diff --git a/src/main/java/org/alliancegenome/curation_api/model/ingest/dto/fms/BiogridOrcFmsDTO.java b/src/main/java/org/alliancegenome/curation_api/model/ingest/dto/fms/BiogridOrcFmsDTO.java index 6742ac634..10ef0c806 100644 --- a/src/main/java/org/alliancegenome/curation_api/model/ingest/dto/fms/BiogridOrcFmsDTO.java +++ b/src/main/java/org/alliancegenome/curation_api/model/ingest/dto/fms/BiogridOrcFmsDTO.java @@ -1,8 +1,5 @@ package org.alliancegenome.curation_api.model.ingest.dto.fms; -import java.util.List; - -import org.alliancegenome.curation_api.model.ingest.dto.CrossReferenceDTO; import org.alliancegenome.curation_api.model.ingest.dto.base.BaseDTO; import lombok.Data; @@ -25,5 +22,4 @@ public class BiogridOrcFmsDTO extends BaseDTO { private Double score5; private String hit; private String source; - private List crossReferenceDtos; } diff --git a/src/main/java/org/alliancegenome/curation_api/model/ingest/dto/fms/BiogridOrcIngestFmsDTO.java b/src/main/java/org/alliancegenome/curation_api/model/ingest/dto/fms/BiogridOrcIngestFmsDTO.java new file mode 100644 index 000000000..45b40347d --- /dev/null +++ b/src/main/java/org/alliancegenome/curation_api/model/ingest/dto/fms/BiogridOrcIngestFmsDTO.java @@ -0,0 +1,16 @@ +package org.alliancegenome.curation_api.model.ingest.dto.fms; + +import java.util.List; + +import org.alliancegenome.curation_api.model.ingest.dto.base.BaseDTO; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +public class BiogridOrcIngestFmsDTO extends BaseDTO { + + private MetaDataFmsDTO metaData; + private List data; +} diff --git a/src/main/java/org/alliancegenome/curation_api/services/BioGridOrcsService.java b/src/main/java/org/alliancegenome/curation_api/services/BioGridOrcsService.java deleted file mode 100644 index 4c401f886..000000000 --- a/src/main/java/org/alliancegenome/curation_api/services/BioGridOrcsService.java +++ /dev/null @@ -1,52 +0,0 @@ -package org.alliancegenome.curation_api.services; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import org.alliancegenome.curation_api.constants.EntityFieldConstants; -import org.alliancegenome.curation_api.dao.CrossReferenceDAO; -import org.alliancegenome.curation_api.dao.SequenceTargetingReagentDAO; -import org.alliancegenome.curation_api.enums.BackendBulkDataProvider; -import org.alliancegenome.curation_api.exceptions.ValidationException; -import org.alliancegenome.curation_api.interfaces.crud.BaseUpsertServiceInterface; -import org.alliancegenome.curation_api.model.entities.CrossReference; -import org.alliancegenome.curation_api.model.entities.SequenceTargetingReagent; -import org.alliancegenome.curation_api.model.ingest.dto.fms.SequenceTargetingReagentFmsDTO; -import org.alliancegenome.curation_api.services.base.SubmittedObjectCrudService; -import org.alliancegenome.curation_api.services.validation.dto.fms.BioGridOrcsFmsDTOValidator; -import org.alliancegenome.curation_api.services.validation.dto.fms.SequenceTargetingReagentFmsDTOValidator; - -import jakarta.annotation.PostConstruct; -import jakarta.enterprise.context.RequestScoped; -import jakarta.inject.Inject; -import jakarta.transaction.Transactional; - -@RequestScoped -public class BioGridOrcsService { - - @Inject BioGridOrcsFmsDTOValidator bioGridOrcsFmsDTOValidator; - @Inject CrossReferenceDAO crossReferenceDAO; - - // @Override - // @PostConstruct - // protected void init() { - // setSQLDao(crossReferenceDAO); - // } - - @Transactional - public CrossReference insert(BioGridOrcsFmsDTOValidator dto, BackendBulkDataProvider dataProvider) throws ValidationException { - // CrossReference crossReference = bioGridOrcsFmsDTOValidator.validateSQTRFmsDTO(dto, dataProvider); - return new CrossReference(); - // return crossReferenceDAO.persist(sqtr); - } - - // public List getIdsByDataProvider(String dataProvider) { - // Map params = new HashMap<>(); - // params.put(EntityFieldConstants.DATA_PROVIDER, dataProvider); - // List ids = sqtrDAO.findIdsByParams(params); - // ids.removeIf(Objects::isNull); - // return ids; - // } -} diff --git a/src/main/java/org/alliancegenome/curation_api/services/validation/dto/fms/BioGridOrcsFmsDTOValidator.java b/src/main/java/org/alliancegenome/curation_api/services/validation/dto/fms/BioGridOrcsFmsDTOValidator.java deleted file mode 100644 index 92c6e9a2e..000000000 --- a/src/main/java/org/alliancegenome/curation_api/services/validation/dto/fms/BioGridOrcsFmsDTOValidator.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.alliancegenome.curation_api.services.validation.dto.fms; - -import java.util.HashMap; - -import org.alliancegenome.curation_api.enums.BackendBulkDataProvider; -import org.alliancegenome.curation_api.model.entities.CrossReference; -import org.alliancegenome.curation_api.model.ingest.dto.fms.BiogridOrcFmsDTO; -import org.alliancegenome.curation_api.response.SearchResponse; - -import jakarta.enterprise.context.RequestScoped; - -@RequestScoped -public class BioGridOrcsFmsDTOValidator { - - //Todo: rename? - public CrossReference validateBioGridOrcsFmsDTO(String referencedCurie){ - // HashMap crossRefParams = new HashMap<>(); - // crossRefParams.put("referencedCurie", referencedCurie); - // crossRefParams.put("displayName", referencedCurie); - // crossRefParams.put("resourceDescriptorPage.id", resourceDescriptorPage.getId()); - - // // Log.debug("--------------crossRefDupSearch----------------"); - // SearchResponse crossRefDupSearch = - // crossRefDAO.findByParams(crossRefParams); - // Log.debug(crossRefDupSearch.getResults().isEmpty()); - - // if(!crossRefDupSearch.getResults().isEmpty()) continue; - - return new CrossReference(); - } - -} diff --git a/src/test/java/org/alliancegenome/curation_api/BiogridOrcBulkUploadFmsITCase.java b/src/test/java/org/alliancegenome/curation_api/BiogridOrcBulkUploadFmsITCase.java new file mode 100644 index 000000000..0c52d02b8 --- /dev/null +++ b/src/test/java/org/alliancegenome/curation_api/BiogridOrcBulkUploadFmsITCase.java @@ -0,0 +1,61 @@ +package org.alliancegenome.curation_api; + +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; + +import org.alliancegenome.curation_api.base.BaseITCase; +import org.alliancegenome.curation_api.resources.TestContainerResource; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.TestMethodOrder; + +import io.quarkus.test.common.QuarkusTestResource; +import io.quarkus.test.junit.QuarkusIntegrationTest; +import io.restassured.RestAssured; +import io.restassured.config.HttpClientConfig; +import io.restassured.config.RestAssuredConfig; + +@QuarkusIntegrationTest +@QuarkusTestResource(TestContainerResource.Initializer.class) +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@DisplayName("612 - Biogrid Orc bulk upload - FMS") +@Order(612) +public class BiogridOrcBulkUploadFmsITCase extends BaseITCase { + + @BeforeEach + public void init() { + RestAssured.config = RestAssuredConfig.config() + .httpClient(HttpClientConfig.httpClientConfig() + .setParam("http.socket.timeout", 100000) + .setParam("http.connection.timeout", 100000)); + } + + private final String biogridOrcBulkPostEndpoint = "/api/cross-reference/bulk/FB/biogridfile"; + private final String biogridOrcTestFilePath = "src/test/resources/bulk/fms/12_biogrid/"; + private final String biogridOrcFindEndpoint = "/api/cross-reference/find?limit=100&page=0"; + + @Test + @Order(1) + public void biogridOrcBulkUploadCheckFields() throws Exception { + + checkSuccessfulBulkLoad(biogridOrcBulkPostEndpoint, biogridOrcTestFilePath + "AF_01_all_fields.json", 1); + + RestAssured.given(). + when(). + header("Content-Type", "application/json"). + body("{}"). + post(biogridOrcFindEndpoint). + then(). + statusCode(200). + body("totalResults", is(1)). + body("results", hasSize(1)). + body("results[0].referencedCurie", is("NCBI_Gene:108101")). + body("results[0].displayName", is("BioGRID CRISPR Screen Cell Line Phenotypes")); + } + +} diff --git a/src/test/resources/bulk/fms/12_biogrid/AF_01_all_fields.json b/src/test/resources/bulk/fms/12_biogrid/AF_01_all_fields.json new file mode 100644 index 000000000..44ffa6cb6 --- /dev/null +++ b/src/test/resources/bulk/fms/12_biogrid/AF_01_all_fields.json @@ -0,0 +1,32 @@ +{ + "data": [ + { + "screenId": 1888, + "identifierId": "108101", + "identifierType": "ENTREZ_GENE", + "officialSymbol": "Fermt3", + "aliases": "C79673|Kindlin3", + "organismId": 10090, + "organismOfficial": "Mus musculus", + "score1": 205.0, + "score2": -2.2, + "score3": 1e-05, + "score4": null, + "score5": null, + "hit": "YES", + "source": "BioGRID ORCS" + } + ], + "metaData": { + "dateProduced": "2024-04-18T20:10:35-07:00", + "dataProvider": { + "crossReference": { + "id": "FB", + "pages": [ + "biogrid/orc" + ] + }, + "type": "curated" + } + } +} From b3b2847717e91b86bf5a5c3a8990f44b05b94429 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Wed, 13 Nov 2024 15:39:23 -0600 Subject: [PATCH 4/7] SCRUM-4513 create biogrid controller --- .../crud/BiogridOrcCrudController.java | 35 +++++++++++++++++++ .../crud/CrossReferenceCrudController.java | 7 ---- .../crud/BiogridOrcCrudInterface.java | 26 ++++++++++++++ .../crud/CrossReferenceCrudInterface.java | 12 ------- 4 files changed, 61 insertions(+), 19 deletions(-) create mode 100644 src/main/java/org/alliancegenome/curation_api/controllers/crud/BiogridOrcCrudController.java create mode 100644 src/main/java/org/alliancegenome/curation_api/interfaces/crud/BiogridOrcCrudInterface.java diff --git a/src/main/java/org/alliancegenome/curation_api/controllers/crud/BiogridOrcCrudController.java b/src/main/java/org/alliancegenome/curation_api/controllers/crud/BiogridOrcCrudController.java new file mode 100644 index 000000000..c7011821c --- /dev/null +++ b/src/main/java/org/alliancegenome/curation_api/controllers/crud/BiogridOrcCrudController.java @@ -0,0 +1,35 @@ +package org.alliancegenome.curation_api.controllers.crud; + +import org.alliancegenome.curation_api.controllers.base.BaseEntityCrudController; +import org.alliancegenome.curation_api.dao.CrossReferenceDAO; +import org.alliancegenome.curation_api.interfaces.crud.BiogridOrcCrudInterface; +import org.alliancegenome.curation_api.jobs.executors.BiogridOrcExecutor; +import org.alliancegenome.curation_api.model.entities.CrossReference; +import org.alliancegenome.curation_api.model.ingest.dto.fms.BiogridOrcIngestFmsDTO; +import org.alliancegenome.curation_api.response.APIResponse; +import org.alliancegenome.curation_api.services.CrossReferenceService; + +import jakarta.annotation.PostConstruct; +import jakarta.enterprise.context.RequestScoped; +import jakarta.inject.Inject; + +@RequestScoped +public class BiogridOrcCrudController extends BaseEntityCrudController implements BiogridOrcCrudInterface { + + @Inject + CrossReferenceService crossReferenceService; + + @Inject + BiogridOrcExecutor biogridOrcExecutor; + + @Override + @PostConstruct + protected void init() { + setService(crossReferenceService); + } + + @Override + public APIResponse updateBiogridOrc(String dataProvider, BiogridOrcIngestFmsDTO biogridOrcData) { + return biogridOrcExecutor.runLoadApi(dataProvider, biogridOrcData.getData()); + } +} diff --git a/src/main/java/org/alliancegenome/curation_api/controllers/crud/CrossReferenceCrudController.java b/src/main/java/org/alliancegenome/curation_api/controllers/crud/CrossReferenceCrudController.java index c9704a1e6..b45cf1ae2 100644 --- a/src/main/java/org/alliancegenome/curation_api/controllers/crud/CrossReferenceCrudController.java +++ b/src/main/java/org/alliancegenome/curation_api/controllers/crud/CrossReferenceCrudController.java @@ -5,8 +5,6 @@ import org.alliancegenome.curation_api.interfaces.crud.CrossReferenceCrudInterface; import org.alliancegenome.curation_api.jobs.executors.BiogridOrcExecutor; import org.alliancegenome.curation_api.model.entities.CrossReference; -import org.alliancegenome.curation_api.model.ingest.dto.fms.BiogridOrcIngestFmsDTO; -import org.alliancegenome.curation_api.response.APIResponse; import org.alliancegenome.curation_api.services.CrossReferenceService; import jakarta.annotation.PostConstruct; @@ -27,9 +25,4 @@ public class CrossReferenceCrudController extends BaseEntityCrudController { + + + @POST + @Path("/bulk/{dataProvider}/biogridfile") + @JsonView(View.FieldsAndLists.class) + APIResponse updateBiogridOrc(@PathParam("dataProvider") String dataProvider, BiogridOrcIngestFmsDTO biogridOrcData); + +} diff --git a/src/main/java/org/alliancegenome/curation_api/interfaces/crud/CrossReferenceCrudInterface.java b/src/main/java/org/alliancegenome/curation_api/interfaces/crud/CrossReferenceCrudInterface.java index f57f7a0a0..5680e1211 100644 --- a/src/main/java/org/alliancegenome/curation_api/interfaces/crud/CrossReferenceCrudInterface.java +++ b/src/main/java/org/alliancegenome/curation_api/interfaces/crud/CrossReferenceCrudInterface.java @@ -2,16 +2,9 @@ import org.alliancegenome.curation_api.interfaces.base.BaseIdCrudInterface; import org.alliancegenome.curation_api.model.entities.CrossReference; -import org.alliancegenome.curation_api.model.ingest.dto.fms.BiogridOrcIngestFmsDTO; -import org.alliancegenome.curation_api.response.APIResponse; -import org.alliancegenome.curation_api.view.View; import org.eclipse.microprofile.openapi.annotations.tags.Tag; -import com.fasterxml.jackson.annotation.JsonView; - -import jakarta.websocket.server.PathParam; import jakarta.ws.rs.Consumes; -import jakarta.ws.rs.POST; import jakarta.ws.rs.Path; import jakarta.ws.rs.Produces; import jakarta.ws.rs.core.MediaType; @@ -21,9 +14,4 @@ @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) public interface CrossReferenceCrudInterface extends BaseIdCrudInterface { - - @POST - @Path("/bulk/{dataProvider}/biogridfile") - @JsonView(View.FieldsAndLists.class) - APIResponse updateBiogridOrc(@PathParam("dataProvider") String dataProvider, BiogridOrcIngestFmsDTO biogridOrcData); } \ No newline at end of file From 0d6fb6313c66aad4597c84ca30a814b3708afe2f Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Wed, 13 Nov 2024 15:55:21 -0600 Subject: [PATCH 5/7] SCRUM-4513 update biogridOrcBulkPostEndpoint --- .../jobs/executors/BiogridOrcExecutor.java | 1 - .../BiogridOrcBulkUploadFmsITCase.java | 33 +++++++++++-------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/alliancegenome/curation_api/jobs/executors/BiogridOrcExecutor.java b/src/main/java/org/alliancegenome/curation_api/jobs/executors/BiogridOrcExecutor.java index c431187de..bd4c42b51 100644 --- a/src/main/java/org/alliancegenome/curation_api/jobs/executors/BiogridOrcExecutor.java +++ b/src/main/java/org/alliancegenome/curation_api/jobs/executors/BiogridOrcExecutor.java @@ -163,7 +163,6 @@ private boolean runLoad(BulkLoadFileHistory history, List biog history.incrementFailed(); ph.progressProcess(); } - history.incrementCompleted(); updateHistory(history); updateExceptions(history); ph.finishProcess(); diff --git a/src/test/java/org/alliancegenome/curation_api/BiogridOrcBulkUploadFmsITCase.java b/src/test/java/org/alliancegenome/curation_api/BiogridOrcBulkUploadFmsITCase.java index 0c52d02b8..f8624b2ce 100644 --- a/src/test/java/org/alliancegenome/curation_api/BiogridOrcBulkUploadFmsITCase.java +++ b/src/test/java/org/alliancegenome/curation_api/BiogridOrcBulkUploadFmsITCase.java @@ -4,6 +4,8 @@ import static org.hamcrest.Matchers.is; import org.alliancegenome.curation_api.base.BaseITCase; +import org.alliancegenome.curation_api.model.entities.ResourceDescriptor; +import org.alliancegenome.curation_api.model.entities.ResourceDescriptorPage; import org.alliancegenome.curation_api.resources.TestContainerResource; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; @@ -35,27 +37,32 @@ public void init() { .setParam("http.connection.timeout", 100000)); } - private final String biogridOrcBulkPostEndpoint = "/api/cross-reference/bulk/FB/biogridfile"; + ResourceDescriptorPage resourceDescriptorPage; + + private final String biogridOrcBulkPostEndpoint = "/api/biogrid-orc/bulk/FB/biogridfile"; private final String biogridOrcTestFilePath = "src/test/resources/bulk/fms/12_biogrid/"; private final String biogridOrcFindEndpoint = "/api/cross-reference/find?limit=100&page=0"; + private void loadRequiredEntities() throws Exception { + ResourceDescriptor rd = createResourceDescriptor("FB"); + createResourceDescriptorPage("biogrid/orcs", "http://test.org", rd); + } + @Test @Order(1) public void biogridOrcBulkUploadCheckFields() throws Exception { - + loadRequiredEntities(); checkSuccessfulBulkLoad(biogridOrcBulkPostEndpoint, biogridOrcTestFilePath + "AF_01_all_fields.json", 1); - RestAssured.given(). - when(). - header("Content-Type", "application/json"). - body("{}"). - post(biogridOrcFindEndpoint). - then(). - statusCode(200). - body("totalResults", is(1)). - body("results", hasSize(1)). - body("results[0].referencedCurie", is("NCBI_Gene:108101")). - body("results[0].displayName", is("BioGRID CRISPR Screen Cell Line Phenotypes")); + RestAssured.given() + .when() + .header("Content-Type", "application/json") + .body("{\"referencedCurie\": \"NCBI_Gene:108101\"}").post(biogridOrcFindEndpoint) + .then().statusCode(200) + .body("totalResults", is(1)) + .body("results", hasSize(1)) + .body("results[0].referencedCurie", is("NCBI_Gene:108101")) + .body("results[0].displayName", is("BioGRID CRISPR Screen Cell Line Phenotypes")); } } From 16be8fb9253b6df5e2d987d6e6e727aa832d4936 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Thu, 14 Nov 2024 12:11:54 -0600 Subject: [PATCH 6/7] SCRUM-4513 add test for duplicate entries --- .../BiogridOrcBulkUploadFmsITCase.java | 33 ++++++++++--- .../12_biogrid/DE_01_duplicate_entries.json | 49 +++++++++++++++++++ 2 files changed, 74 insertions(+), 8 deletions(-) create mode 100644 src/test/resources/bulk/fms/12_biogrid/DE_01_duplicate_entries.json diff --git a/src/test/java/org/alliancegenome/curation_api/BiogridOrcBulkUploadFmsITCase.java b/src/test/java/org/alliancegenome/curation_api/BiogridOrcBulkUploadFmsITCase.java index f8624b2ce..54e1a1f4b 100644 --- a/src/test/java/org/alliancegenome/curation_api/BiogridOrcBulkUploadFmsITCase.java +++ b/src/test/java/org/alliancegenome/curation_api/BiogridOrcBulkUploadFmsITCase.java @@ -55,14 +55,31 @@ public void biogridOrcBulkUploadCheckFields() throws Exception { checkSuccessfulBulkLoad(biogridOrcBulkPostEndpoint, biogridOrcTestFilePath + "AF_01_all_fields.json", 1); RestAssured.given() - .when() - .header("Content-Type", "application/json") - .body("{\"referencedCurie\": \"NCBI_Gene:108101\"}").post(biogridOrcFindEndpoint) - .then().statusCode(200) - .body("totalResults", is(1)) - .body("results", hasSize(1)) - .body("results[0].referencedCurie", is("NCBI_Gene:108101")) - .body("results[0].displayName", is("BioGRID CRISPR Screen Cell Line Phenotypes")); + .when() + .header("Content-Type", "application/json") + .body("{\"referencedCurie\": \"NCBI_Gene:108101\"}").post(biogridOrcFindEndpoint) + .then().statusCode(200) + .body("totalResults", is(1)) + .body("results", hasSize(1)) + .body("results[0].referencedCurie", is("NCBI_Gene:108101")) + .body("results[0].displayName", is("BioGRID CRISPR Screen Cell Line Phenotypes")); + } + + @Test + @Order(2) + public void biogridOrcBulkUploadDuplicateEntries() throws Exception { + checkBulkLoadRecordCounts(biogridOrcBulkPostEndpoint, biogridOrcTestFilePath + "DE_01_duplicate_entries.json", + "Records", 2, 0, 1, 0); + + RestAssured.given() + .when() + .header("Content-Type", "application/json") + .body("{\"referencedCurie\": \"NCBI_Gene:100001\"}").post(biogridOrcFindEndpoint) + .then().statusCode(200) + .body("totalResults", is(1)) + .body("results", hasSize(1)) + .body("results[0].referencedCurie", is("NCBI_Gene:100001")) + .body("results[0].displayName", is("BioGRID CRISPR Screen Cell Line Phenotypes")); } } diff --git a/src/test/resources/bulk/fms/12_biogrid/DE_01_duplicate_entries.json b/src/test/resources/bulk/fms/12_biogrid/DE_01_duplicate_entries.json new file mode 100644 index 000000000..677cb545b --- /dev/null +++ b/src/test/resources/bulk/fms/12_biogrid/DE_01_duplicate_entries.json @@ -0,0 +1,49 @@ +{ + "data": [ + { + "screenId": 1888, + "identifierId": "100001", + "identifierType": "ENTREZ_GENE", + "officialSymbol": "Fermt3", + "aliases": "C79673|Kindlin3", + "organismId": 10090, + "organismOfficial": "Mus musculus", + "score1": 205.0, + "score2": -2.2, + "score3": 1e-05, + "score4": null, + "score5": null, + "hit": "YES", + "source": "BioGRID ORCS" + }, + { + "screenId": 1888, + "identifierId": "100001", + "identifierType": "ENTREZ_GENE", + "officialSymbol": "Fermt3", + "aliases": "C79673|Kindlin3", + "organismId": 10090, + "organismOfficial": "Mus musculus", + "score1": 205.0, + "score2": -2.2, + "score3": 1e-05, + "score4": null, + "score5": null, + "hit": "YES", + "source": "BioGRID ORCS" + } + ], + "metaData": { + "dateProduced": "2024-04-18T20:10:35-07:00", + "dataProvider": { + "crossReference": { + "id": "FB", + "pages": [ + "biogrid/orc" + ] + }, + "type": "curated" + } + } + } + \ No newline at end of file From e2180ced2ebf8e73184f7584ddf780a822ce3ce5 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Thu, 14 Nov 2024 16:18:44 -0600 Subject: [PATCH 7/7] SCRUM-4513 remove unused import --- .../controllers/crud/CrossReferenceCrudController.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/java/org/alliancegenome/curation_api/controllers/crud/CrossReferenceCrudController.java b/src/main/java/org/alliancegenome/curation_api/controllers/crud/CrossReferenceCrudController.java index b45cf1ae2..48de548e8 100644 --- a/src/main/java/org/alliancegenome/curation_api/controllers/crud/CrossReferenceCrudController.java +++ b/src/main/java/org/alliancegenome/curation_api/controllers/crud/CrossReferenceCrudController.java @@ -3,7 +3,6 @@ import org.alliancegenome.curation_api.controllers.base.BaseEntityCrudController; import org.alliancegenome.curation_api.dao.CrossReferenceDAO; import org.alliancegenome.curation_api.interfaces.crud.CrossReferenceCrudInterface; -import org.alliancegenome.curation_api.jobs.executors.BiogridOrcExecutor; import org.alliancegenome.curation_api.model.entities.CrossReference; import org.alliancegenome.curation_api.services.CrossReferenceService; @@ -17,9 +16,6 @@ public class CrossReferenceCrudController extends BaseEntityCrudController