Merge pull request #436 from alliance-genome/SCRUM-1371-SCRUM-1396-ge…

…ne-and-agm-removal SCRUM-1371 + SCRUM-1396: non-loaded gene and agm removal
alliance-genome · Apr 27, 2022 · 4b5140f · 4b5140f
2 parents 0b272e3 + b5efe2b
commit 4b5140f
Show file tree

Hide file tree

Showing 55 changed files with 603 additions and 446 deletions.
diff --git a/docker/run_es b/docker/run_es
@@ -1 +1 @@
-docker run -d --net curation -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" --name elasticsearch docker.elastic.co/elasticsearch/elasticsearch:7.10.2
+docker run -d --net curation -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" -e "ES_JAVA_OPTS=-Xmx4g -Xms4g" --name elasticsearch docker.elastic.co/elasticsearch/elasticsearch:7.10.2
diff --git a/src/main/java/org/alliancegenome/curation_api/base/services/BaseCrudService.java b/src/main/java/org/alliancegenome/curation_api/base/services/BaseCrudService.java
@@ -8,7 +8,6 @@
 import org.alliancegenome.curation_api.auth.AuthenticatedUser;
 import org.alliancegenome.curation_api.base.dao.*;
 import org.alliancegenome.curation_api.base.entity.BaseEntity;
-import org.alliancegenome.curation_api.model.entities.AffectedGenomicModel;
 import org.alliancegenome.curation_api.model.entities.Person;
 import org.alliancegenome.curation_api.model.input.Pagination;
 import org.alliancegenome.curation_api.response.*;

diff --git a/.../org/alliancegenome/curation_api/controllers/crud/AffectedGenomicModelCrudController.java b/.../org/alliancegenome/curation_api/controllers/crud/AffectedGenomicModelCrudController.java
@@ -3,21 +3,14 @@
 import javax.annotation.PostConstruct;
 import javax.enterprise.context.RequestScoped;
 import javax.inject.Inject;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-
-import com.fasterxml.jackson.annotation.JsonView;
-import org.alliancegenome.curation_api.auth.Secured;
 import org.alliancegenome.curation_api.base.controllers.BaseCrudController;
 import org.alliancegenome.curation_api.dao.AffectedGenomicModelDAO;
 import org.alliancegenome.curation_api.interfaces.crud.AffectedGenomicModelCrudInterface;
 import org.alliancegenome.curation_api.jobs.executors.AgmFmsExecutor;
 import org.alliancegenome.curation_api.model.entities.AffectedGenomicModel;
 import org.alliancegenome.curation_api.model.ingest.fms.dto.AffectedGenomicModelMetaDataFmsDTO;
 import org.alliancegenome.curation_api.response.APIResponse;
-import org.alliancegenome.curation_api.response.ObjectResponse;
 import org.alliancegenome.curation_api.services.AffectedGenomicModelService;
-import org.alliancegenome.curation_api.view.View;
 
 @RequestScoped
 public class AffectedGenomicModelCrudController extends BaseCrudController<AffectedGenomicModelService, AffectedGenomicModel, AffectedGenomicModelDAO> implements AffectedGenomicModelCrudInterface {

diff --git a/...n/java/org/alliancegenome/curation_api/controllers/crud/VocabularyTermCrudController.java b/...n/java/org/alliancegenome/curation_api/controllers/crud/VocabularyTermCrudController.java
@@ -7,10 +7,8 @@
 import org.alliancegenome.curation_api.base.controllers.BaseCrudController;
 import org.alliancegenome.curation_api.dao.VocabularyTermDAO;
 import org.alliancegenome.curation_api.interfaces.crud.VocabularyTermCrudInterface;
-import org.alliancegenome.curation_api.model.entities.DiseaseAnnotation;
 import org.alliancegenome.curation_api.model.entities.VocabularyTerm;
 import org.alliancegenome.curation_api.response.ObjectResponse;
-import org.alliancegenome.curation_api.response.SearchResponse;
 import org.alliancegenome.curation_api.services.VocabularyTermService;
 
 @RequestScoped

diff --git a/src/main/java/org/alliancegenome/curation_api/dao/AffectedGenomicModelDAO.java b/src/main/java/org/alliancegenome/curation_api/dao/AffectedGenomicModelDAO.java
@@ -1,6 +1,9 @@
 package org.alliancegenome.curation_api.dao;
 
+import java.util.List;
+
 import javax.enterprise.context.ApplicationScoped;
+import javax.persistence.Query;
 
 import org.alliancegenome.curation_api.base.dao.BaseSQLDAO;
 import org.alliancegenome.curation_api.model.entities.AffectedGenomicModel;
@@ -11,4 +14,10 @@ public class AffectedGenomicModelDAO extends BaseSQLDAO<AffectedGenomicModel> {
     protected AffectedGenomicModelDAO() {
         super(AffectedGenomicModel.class);
     }
+
+    public List<String> findAllAnnotationIds(String taxonID) {
+        Query jpqlQuery = entityManager.createQuery("SELECT annotation.curie FROM AffectedGenomicModel annotation WHERE annotation.taxon.curie=:taxonId");
+        jpqlQuery.setParameter("taxonId", taxonID);
+        return (List<String>) jpqlQuery.getResultList();
+    }
 }
diff --git a/src/main/java/org/alliancegenome/curation_api/dao/GeneDAO.java b/src/main/java/org/alliancegenome/curation_api/dao/GeneDAO.java
@@ -1,6 +1,9 @@
 package org.alliancegenome.curation_api.dao;
 
+import java.util.List;
+
 import javax.enterprise.context.ApplicationScoped;
+import javax.persistence.Query;
 
 import org.alliancegenome.curation_api.base.dao.BaseSQLDAO;
 import org.alliancegenome.curation_api.model.entities.Gene;
@@ -16,4 +19,10 @@ public Gene getByIdOrCurie(String id) {
         return find(id);
     }
 
+    public List<String> findAllAnnotationIds(String taxonID) {
+        Query jpqlQuery = entityManager.createQuery("SELECT annotation.curie FROM Gene annotation WHERE annotation.taxon.curie=:taxonId");
+        jpqlQuery.setParameter("taxonId", taxonID);
+        return (List<String>) jpqlQuery.getResultList();
+    }
+
 }
diff --git a/src/main/java/org/alliancegenome/curation_api/jobs/executors/AgmFmsExecutor.java b/src/main/java/org/alliancegenome/curation_api/jobs/executors/AgmFmsExecutor.java
@@ -1,25 +1,32 @@
 package org.alliancegenome.curation_api.jobs.executors;
 
 import java.io.FileInputStream;
+import java.util.*;
+import java.util.stream.Collectors;
 import java.util.zip.GZIPInputStream;
 
 import javax.enterprise.context.ApplicationScoped;
 import javax.inject.Inject;
 
+import org.alliancegenome.curation_api.dao.AffectedGenomicModelDAO;
 import org.alliancegenome.curation_api.exceptions.ObjectUpdateException;
 import org.alliancegenome.curation_api.exceptions.ObjectUpdateException.ObjectUpdateExceptionData;
+import org.alliancegenome.curation_api.model.entities.AffectedGenomicModel;
 import org.alliancegenome.curation_api.model.entities.bulkloads.*;
 import org.alliancegenome.curation_api.model.ingest.fms.dto.*;
 import org.alliancegenome.curation_api.response.*;
 import org.alliancegenome.curation_api.services.AffectedGenomicModelService;
 import org.alliancegenome.curation_api.util.ProcessDisplayHelper;
+import org.apache.commons.collections4.ListUtils;
 
 import lombok.extern.jbosslog.JBossLog;
 
 @JBossLog
 @ApplicationScoped
 public class AgmFmsExecutor extends LoadFileExecutor {
 
+    @Inject AffectedGenomicModelDAO affectedGenomicModelDAO;
+
     @Inject AffectedGenomicModelService affectedGenomicModelService;
 
     public void runLoad(BulkLoadFile bulkLoadFile) {
@@ -38,25 +45,55 @@ public void runLoad(BulkLoadFile bulkLoadFile) {
 
     // Gets called from the API directly
     public APIResponse runLoad(AffectedGenomicModelMetaDataFmsDTO agmData) {
+        List<String> taxonIDs = agmData.getData().stream()
+                                 .map( agmDTO -> agmDTO.getTaxonId() ).distinct().collect( Collectors.toList() );
+
+        List<String> annotationsIdsBefore = new ArrayList<String>();
+        for(String taxonID: taxonIDs) {
+            List<String> annotationIds = affectedGenomicModelDAO.findAllAnnotationIds(taxonID);
+            log.debug("runLoad: Before: taxonID " + taxonID + " " + annotationIds.size());
+            annotationsIdsBefore.addAll(annotationIds);
+        }
+
+        log.debug("runLoad: Before: total " + annotationsIdsBefore.size());
+        List<String> annotationsIdsLoaded = new ArrayList<>();
 
         BulkLoadFileHistory history = new BulkLoadFileHistory(agmData.getData().size());
 
         ProcessDisplayHelper ph = new ProcessDisplayHelper(10000);
         ph.startProcess("AGM FMS DTO Update", agmData.getData().size());
-        for(AffectedGenomicModelFmsDTO agm: agmData.getData()) {
+        for(AffectedGenomicModelFmsDTO agmDTO: agmData.getData()) {
             try {
-                affectedGenomicModelService.processUpdate(agm);
+                AffectedGenomicModel annotation = affectedGenomicModelService.processUpdate(agmDTO);
                 history.incrementCompleted();
+                annotationsIdsLoaded.add(annotation.getCurie());
             } catch (ObjectUpdateException e) {
                 addException(history, e.getData());
             } catch (Exception e) {
-                addException(history, new ObjectUpdateExceptionData(agm, e.getMessage()));
+                addException(history, new ObjectUpdateExceptionData(agmDTO, e.getMessage()));
             }
 
             ph.progressProcess();
         }
         ph.finishProcess();
-
+
+        log.debug("runLoad: Loaded: " + taxonIDs.toString() + " " + annotationsIdsLoaded.size());
+
+        List<String> distinctLoaded = annotationsIdsLoaded.stream().distinct().collect(Collectors.toList());
+        log.debug("runLoad: Distinct loaded: " + taxonIDs.toString() + " " + distinctLoaded.size());
+
+        List<String> curiesToRemove = ListUtils.subtract(annotationsIdsBefore, distinctLoaded);
+        log.debug("runLoad: Remove: " + taxonIDs.toString() + " " + curiesToRemove.size());
+
+        for (String curie : curiesToRemove) {
+            SearchResponse<AffectedGenomicModel> agm = affectedGenomicModelDAO.findByField("curie", curie);
+            if (agm != null && agm.getTotalResults() == 1) {
+                affectedGenomicModelDAO.remove(agm.getResults().get(0).getCurie());
+            } else {
+                log.error("Failed getting annotation: " + curie);
+            }
+        }
+
         return new LoadHistoryResponce(history);
     }
 }
diff --git a/src/main/java/org/alliancegenome/curation_api/jobs/executors/AlleleFmsExecutor.java b/src/main/java/org/alliancegenome/curation_api/jobs/executors/AlleleFmsExecutor.java
@@ -9,7 +9,6 @@
 import javax.inject.Inject;
 
 import org.alliancegenome.curation_api.dao.AlleleDAO;
-import org.alliancegenome.curation_api.enums.BackendBulkDataType;
 import org.alliancegenome.curation_api.exceptions.ObjectUpdateException;
 import org.alliancegenome.curation_api.exceptions.ObjectUpdateException.ObjectUpdateExceptionData;
 import org.alliancegenome.curation_api.model.entities.Allele;

diff --git a/src/main/java/org/alliancegenome/curation_api/jobs/executors/GeneFmsExecutor.java b/src/main/java/org/alliancegenome/curation_api/jobs/executors/GeneFmsExecutor.java
@@ -1,66 +1,103 @@
 package org.alliancegenome.curation_api.jobs.executors;
 
 import java.io.FileInputStream;
+import java.util.*;
+import java.util.stream.Collectors;
 import java.util.zip.GZIPInputStream;
 
 import javax.enterprise.context.ApplicationScoped;
 import javax.inject.Inject;
 
+import org.alliancegenome.curation_api.dao.GeneDAO;
 import org.alliancegenome.curation_api.exceptions.ObjectUpdateException;
 import org.alliancegenome.curation_api.exceptions.ObjectUpdateException.ObjectUpdateExceptionData;
+import org.alliancegenome.curation_api.model.entities.Gene;
 import org.alliancegenome.curation_api.model.entities.bulkloads.*;
 import org.alliancegenome.curation_api.model.ingest.fms.dto.*;
 import org.alliancegenome.curation_api.response.*;
 import org.alliancegenome.curation_api.services.GeneService;
 import org.alliancegenome.curation_api.util.ProcessDisplayHelper;
+import org.apache.commons.collections4.ListUtils;
 
 import lombok.extern.jbosslog.JBossLog;
 
 @JBossLog
 @ApplicationScoped
 public class GeneFmsExecutor extends LoadFileExecutor {
 
+    @Inject GeneDAO geneDAO;
+
     @Inject GeneService geneService;
 
 
     public void runLoad(BulkLoadFile bulkLoadFile) {
-        
+
         try {
             GeneMetaDataFmsDTO geneData = mapper.readValue(new GZIPInputStream(new FileInputStream(bulkLoadFile.getLocalFilePath())), GeneMetaDataFmsDTO.class);
             bulkLoadFile.setRecordCount(geneData.getData().size());
             bulkLoadFileDAO.merge(bulkLoadFile);
-            
+
             trackHistory(runLoad(geneData), bulkLoadFile);
-        
+
         } catch (Exception e) {
             e.printStackTrace();
         }
     }
 
     // Gets called from the API directly
     public APIResponse runLoad(GeneMetaDataFmsDTO geneData) {
+        List<String> taxonIDs = geneData.getData().stream()
+                                 .map( geneDTO -> geneDTO.getBasicGeneticEntity().getTaxonId() ).distinct().collect( Collectors.toList() );
+
+        List<String> annotationsIdsBefore = new ArrayList<String>();
+        for(String taxonID: taxonIDs) {
+            List<String> annotationIds = geneDAO.findAllAnnotationIds(taxonID);
+            log.debug("runLoad: Before: taxonID " + taxonID + " " + annotationIds.size());
+            annotationsIdsBefore.addAll(annotationIds);
+        }
+
+        log.debug("runLoad: Before: total " + annotationsIdsBefore.size());
+        List<String> annotationsIdsLoaded = new ArrayList<>();
 
         BulkLoadFileHistory history = new BulkLoadFileHistory(geneData.getData().size());
 
         ProcessDisplayHelper ph = new ProcessDisplayHelper(10000);
 
         ph.startProcess("Gene FMS DTO Update", geneData.getData().size());
 
-        for(GeneFmsDTO gene: geneData.getData()) {
+        for(GeneFmsDTO geneDTO: geneData.getData()) {
 
             try {
-                geneService.processUpdate(gene);
+                Gene annotation = geneService.processUpdate(geneDTO);
                 history.incrementCompleted();
+                annotationsIdsLoaded.add(annotation.getCurie());
             } catch (ObjectUpdateException e) {
                 addException(history, e.getData());
             } catch (Exception e) {
-                addException(history, new ObjectUpdateExceptionData(gene, e.getMessage()));
+                addException(history, new ObjectUpdateExceptionData(geneDTO, e.getMessage()));
             }
 
             ph.progressProcess();
         }
         ph.finishProcess();
-
+
+        log.debug("runLoad: Loaded: " + taxonIDs.toString() + " " + annotationsIdsLoaded.size());
+
+        List<String> distinctLoaded = annotationsIdsLoaded.stream().distinct().collect(Collectors.toList());
+        log.debug("runLoad: Distinct loaded: " + taxonIDs.toString() + " " + distinctLoaded.size());
+
+        List<String> curiesToRemove = ListUtils.subtract(annotationsIdsBefore, distinctLoaded);
+        log.debug("runLoad: Remove: " + taxonIDs.toString() + " " + curiesToRemove.size());
+
+        for (String curie : curiesToRemove) {
+            SearchResponse<Gene> gene = geneDAO.findByField("curie", curie);
+            if (gene != null && gene.getTotalResults() == 1) {
+                geneDAO.remove(gene.getResults().get(0).getCurie());
+            } else {
+                log.error("Failed getting annotation: " + curie);
+            }
+        }
+
         return new LoadHistoryResponce(history);
     }
 }
diff --git a/src/main/java/org/alliancegenome/curation_api/model/entities/AGMDiseaseAnnotation.java b/src/main/java/org/alliancegenome/curation_api/model/entities/AGMDiseaseAnnotation.java
@@ -23,7 +23,9 @@ public class AGMDiseaseAnnotation extends DiseaseAnnotation {
 
     @IndexedEmbedded(includeDepth = 1)
     @IndexingDependency(reindexOnUpdate = ReindexOnUpdate.SHALLOW)
-    @ManyToOne
+    @ManyToOne(cascade = {})
+    @org.hibernate.annotations.OnDelete(action = org.hibernate.annotations.OnDeleteAction.CASCADE)
+    @JoinColumn(foreignKey = @ForeignKey(name="fk_agmdasubject"))
     @JsonView({View.FieldsOnly.class})
     private AffectedGenomicModel subject;
 

diff --git a/src/main/java/org/alliancegenome/curation_api/model/entities/DiseaseAnnotation.java b/src/main/java/org/alliancegenome/curation_api/model/entities/DiseaseAnnotation.java
@@ -19,8 +19,6 @@
 import org.hibernate.search.mapper.pojo.mapping.definition.annotation.*;
 import com.fasterxml.jackson.annotation.*;
 import com.fasterxml.jackson.annotation.JsonSubTypes.Type;
-import org.hibernate.search.mapper.pojo.bridge.PropertyBridge;
-import org.hibernate.search.mapper.pojo.bridge.binding.PropertyBindingContext;
 import org.hibernate.search.mapper.pojo.bridge.mapping.annotation.PropertyBinderRef;
 
 import lombok.*;

diff --git a/src/main/java/org/alliancegenome/curation_api/model/entities/GeneDiseaseAnnotation.java b/src/main/java/org/alliancegenome/curation_api/model/entities/GeneDiseaseAnnotation.java
@@ -23,7 +23,9 @@ public class GeneDiseaseAnnotation extends DiseaseAnnotation {
 
     @IndexedEmbedded(includeDepth = 1)
     @IndexingDependency(reindexOnUpdate = ReindexOnUpdate.SHALLOW)
-    @ManyToOne
+    @ManyToOne(cascade = {})
+    @org.hibernate.annotations.OnDelete(action = org.hibernate.annotations.OnDeleteAction.CASCADE)
+    @JoinColumn(foreignKey = @ForeignKey(name="fk_genedasubject"))
     @JsonView({View.FieldsOnly.class})
     private Gene subject;
 

diff --git a/src/main/java/org/alliancegenome/curation_api/services/AffectedGenomicModelService.java b/src/main/java/org/alliancegenome/curation_api/services/AffectedGenomicModelService.java
@@ -87,7 +87,7 @@ public ObjectResponse<AffectedGenomicModel> update(AffectedGenomicModel uiEntity
 
 
     @Transactional
-    public void processUpdate(AffectedGenomicModelFmsDTO agm) throws ObjectUpdateException {
+    public AffectedGenomicModel processUpdate(AffectedGenomicModelFmsDTO agm) throws ObjectUpdateException {
         // TODO: add loading of components
         // TODO: add loading of sequenceTargetingReagents
         // TODO: add loading of parentalPopulations
@@ -114,6 +114,8 @@ public void processUpdate(AffectedGenomicModelFmsDTO agm) throws ObjectUpdateExc
         handleSecondaryIds(agm, dbAgm);
 
         affectedGenomicModelDAO.persist(dbAgm);
+
+        return dbAgm;
     }
 
     private void handleCrossReference(AffectedGenomicModelFmsDTO agm, AffectedGenomicModel dbAgm) {

diff --git a/src/main/java/org/alliancegenome/curation_api/services/GeneService.java b/src/main/java/org/alliancegenome/curation_api/services/GeneService.java
@@ -70,7 +70,7 @@ public Gene getByIdOrCurie(String id) {
     }
 
     @Transactional
-    public void processUpdate(GeneFmsDTO gene) throws ObjectUpdateException {
+    public Gene processUpdate(GeneFmsDTO gene) throws ObjectUpdateException {
         //log.info("processUpdate Gene: ");
 
         validateGeneDTO(gene);
@@ -100,6 +100,7 @@ public void processUpdate(GeneFmsDTO gene) throws ObjectUpdateException {
 
         geneDAO.persist(g);
 
+        return g;
     }
 
     private void handleSecondaryIds(GeneFmsDTO geneFmsDTO, Gene gene) {

diff --git a/src/main/resources/db/migration/v0.4.0.1__agr_curation_api.sql b/src/main/resources/db/migration/v0.4.0.1__agr_curation_api.sql
@@ -0,0 +1,6 @@
+-- Need to drop index to allow hibernate to recreate it with new definition (as hibernate doesn't replace constraints by itself)
+-- This change does require the application to be restarted after initial start-up (so hibernate would create the new constraint)!
+ALTER TABLE "genediseaseannotation"
+	DROP CONSTRAINT "fk8xs26m9hfc38nmy7gvu3cec3t";
+ALTER TABLE "agmdiseaseannotation"
+	DROP CONSTRAINT "fklvr4o1waqclvbktjmyg6x25ls";