Feat/met 6060 pass debuplication reason to ecloud and restore back during tombstoning

metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/DepublicationReason.java

@@ -0,0 +1,32 @@
+package eu.europeana.metis.utils;
+
+/**
+ * Enum for depublication reason.
+ */
+public enum DepublicationReason {
+
+  BROKEN_MEDIA_LINKS("Broken media links", "http://data.europeana.eu/vocabulary/depublicationReason/contentTier0"),
+  GDPR("GDPR", "http://data.europeana.eu/vocabulary/depublicationReason/gdpr"),
+  PERMISSION_ISSUES("Permission issues", "http://data.europeana.eu/vocabulary/depublicationReason/noPermission"),
+  SENSITIVE_CONTENT("Sensitive content", "http://data.europeana.eu/vocabulary/depublicationReason/sensitiveContent"),
+  REMOVED_DATA_AT_SOURCE("Removed data at source", "http://data.europeana.eu/vocabulary/depublicationReason/sourceRemoval"),
+  GENERIC("Generic", "http://data.europeana.eu/vocabulary/depublicationReason/generic"),
+  UNKNOWN("Unknown", "http://data.europeana.eu/vocabulary/depublicationReason/unknown");
+
+  private final String valueAsString;
+  private final String url;
+
+  DepublicationReason(String valueAsString, String url) {
+    this.valueAsString = valueAsString;
+    this.url = url;
+  }
+
+  @Override
+  public String toString(){
+    return valueAsString;
+  }
+
+  public String getUrl() {
+    return url;
+  }
+}

metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/dataset/DepublishRecordId.java

@@ -6,7 +6,7 @@
 import dev.morphia.annotations.Index;
 import dev.morphia.annotations.IndexOptions;
 import dev.morphia.annotations.Indexes;
-import eu.europeana.metis.core.workflow.plugins.DepublicationReason;
+import eu.europeana.metis.utils.DepublicationReason;
 import eu.europeana.metis.mongo.model.HasMongoObjectId;
 import java.time.Instant;
 import org.bson.types.ObjectId;

metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/rest/DepublishRecordIdView.java

@@ -3,7 +3,7 @@
 import com.fasterxml.jackson.databind.annotation.JsonSerialize;
 import eu.europeana.metis.core.common.JavaTimeSerialization.IsoInstantSerializer;
 import eu.europeana.metis.core.dataset.DepublishRecordId;
-import eu.europeana.metis.core.workflow.plugins.DepublicationReason;
+import eu.europeana.metis.utils.DepublicationReason;
 import java.time.Instant;
 
 /**

metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/DepublishPlugin.java

@@ -63,6 +63,8 @@ public DpsTask prepareDpsTask(String datasetId,
         extraParameters.put(PluginParameterKeys.RECORD_IDS_TO_DEPUBLISH, recordIdList);
       }
     }
+    //TODO: 2024-09-24 - Update below key with the PluginParameterKeys equivalent when it's available
+    extraParameters.put("DEPUBLICATION_REASON", getPluginMetadata().getDepublicationReason().name());
     DpsTask dpsTask = new DpsTask();
     dpsTask.setParameters(extraParameters);
     return dpsTask;

metis-core/metis-core-common/src/main/java/eu/europeana/metis/core/workflow/plugins/DepublishPluginMetadata.java

@@ -1,6 +1,7 @@
 package eu.europeana.metis.core.workflow.plugins;
 
 
+import eu.europeana.metis.utils.DepublicationReason;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Optional;

metis-core/metis-core-common/src/test/java/eu/europeana/metis/core/workflow/plugins/DepublishPluginMetadataTest.java

@@ -2,6 +2,7 @@
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
+import eu.europeana.metis.utils.DepublicationReason;
 import java.util.Set;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;

metis-core/metis-core-rest/src/main/java/eu/europeana/metis/core/rest/controller/DepublishRecordIdController.java

@@ -9,7 +9,7 @@
 import eu.europeana.metis.core.util.DepublishRecordIdSortField;
 import eu.europeana.metis.core.util.SortDirection;
 import eu.europeana.metis.core.workflow.WorkflowExecution;
-import eu.europeana.metis.core.workflow.plugins.DepublicationReason;
+import eu.europeana.metis.utils.DepublicationReason;
 import eu.europeana.metis.exception.BadContentException;
 import eu.europeana.metis.exception.GenericMetisException;
 import eu.europeana.metis.exception.UserUnauthorizedException;

metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/dao/DepublishRecordIdDao.java

@@ -17,7 +17,7 @@
 import eu.europeana.metis.core.rest.RequestLimits;
 import eu.europeana.metis.core.util.DepublishRecordIdSortField;
 import eu.europeana.metis.core.util.SortDirection;
-import eu.europeana.metis.core.workflow.plugins.DepublicationReason;
+import eu.europeana.metis.utils.DepublicationReason;
 import eu.europeana.metis.exception.BadContentException;
 import java.time.Instant;
 import java.util.ArrayList;

metis-core/metis-core-service/src/main/java/eu/europeana/metis/core/service/DepublishRecordIdService.java

@@ -12,7 +12,7 @@
 import eu.europeana.metis.core.util.SortDirection;
 import eu.europeana.metis.core.workflow.Workflow;
 import eu.europeana.metis.core.workflow.WorkflowExecution;
-import eu.europeana.metis.core.workflow.plugins.DepublicationReason;
+import eu.europeana.metis.utils.DepublicationReason;
 import eu.europeana.metis.core.workflow.plugins.DepublishPluginMetadata;
 import eu.europeana.metis.exception.BadContentException;
 import eu.europeana.metis.exception.GenericMetisException;

metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/dao/TestDepublishRecordIdDao.java

@@ -19,7 +19,7 @@
 import eu.europeana.metis.core.util.DepublishRecordIdSortField;
 import eu.europeana.metis.core.util.SortDirection;
 import eu.europeana.metis.core.utils.TestObjectFactory;
-import eu.europeana.metis.core.workflow.plugins.DepublicationReason;
+import eu.europeana.metis.utils.DepublicationReason;
 import eu.europeana.metis.exception.BadContentException;
 import eu.europeana.metis.mongo.embedded.EmbeddedLocalhostMongo;
 import java.time.Instant;

metis-core/metis-core-service/src/test/java/eu/europeana/metis/core/service/TestDepublishRecordIdService.java

@@ -26,7 +26,7 @@
 import eu.europeana.metis.core.utils.TestObjectFactory;
 import eu.europeana.metis.core.workflow.Workflow;
 import eu.europeana.metis.core.workflow.WorkflowExecution;
-import eu.europeana.metis.core.workflow.plugins.DepublicationReason;
+import eu.europeana.metis.utils.DepublicationReason;
 import eu.europeana.metis.exception.GenericMetisException;
 import java.util.List;
 import java.util.Set;

metis-indexing/src/main/java/eu/europeana/indexing/AbstractConnectionProvider.java

@@ -56,7 +56,7 @@ default void triggerFlushOfPendingChanges(boolean blockUntilComplete)
    * @return A dataset remover.
    */
   default IndexedRecordAccess getIndexedRecordAccess() {
-    return new IndexedRecordAccess(getRecordDao(), getSolrClient());
+    return new IndexedRecordAccess(getRecordDao(), getTombstoneRecordDao(), getSolrClient());
   }
 
   /**

metis-indexing/src/main/java/eu/europeana/indexing/IndexedRecordAccess.java

@@ -35,16 +35,19 @@ public class IndexedRecordAccess {
   private static final String ABOUT_FIELD = "about";
 
   private final RecordDao recordDao;
+  private final RecordDao tombstoneDao;
   private final SolrClient solrServer;
 
   /**
    * Constructor.
    *
    * @param recordDao the mongo dao for connecting with the Mongo records database.
+   * @param tombstoneDao the mongo dao for connecting with the Mongo tombstone records database.
    * @param solrServer The Solr server connection.
    */
-  IndexedRecordAccess(RecordDao recordDao, SolrClient solrServer) {
+  IndexedRecordAccess(RecordDao recordDao, RecordDao tombstoneDao, SolrClient solrServer) {
     this.recordDao = recordDao;
+    this.tombstoneDao = tombstoneDao;
     this.solrServer = solrServer;
   }
 
@@ -60,7 +63,7 @@ public long countRecords(String datasetId) {
   }
 
   /**
-   * Get fullbean from database give the rdf about.
+   * Get fullbean from database given an rdf about.
    * @param rdfAbout the rdf about
    * @return the fullbean
    */
@@ -69,6 +72,16 @@ public FullBeanImpl getFullbean(String rdfAbout) {
     return datastore.find(FullBeanImpl.class).filter(Filters.eq(ABOUT_FIELD, rdfAbout)).first();
   }
 
+  /**
+   * Get fullbean from tombstone database given an rdf about.
+   * @param rdfAbout the rdf about
+   * @return the fullbean
+   */
+  public FullBeanImpl getTombstoneFullbean(String rdfAbout) {
+    final Datastore datastore = tombstoneDao.getDatastore();
+    return datastore.find(FullBeanImpl.class).filter(Filters.eq(ABOUT_FIELD, rdfAbout)).first();
+  }
+
   /**
    * Removes the record with the given rdf:about value. Also removes any associated entities (i.e. those entities that are always
    * part of only one record and the removal of which can not invalidate references from other records):

metis-indexing/src/main/java/eu/europeana/indexing/Indexer.java

@@ -1,5 +1,7 @@
 package eu.europeana.indexing;
 
+import eu.europeana.corelib.solr.bean.impl.FullBeanImpl;
+import eu.europeana.metis.utils.DepublicationReason;
 import java.io.Closeable;
 import java.io.InputStream;
 import java.util.Date;
@@ -179,15 +181,35 @@ TierResults indexAndGetTierCalculations(InputStream recordContent,
    */
   boolean remove(String rdfAbout) throws IndexingException;
 
+  /**
+   * Get a tombstone record given an rdf about.
+   * @param rdfAbout the rdf about
+   * @return the tombstone record or else null
+   */
+  FullBeanImpl getTombstone(String rdfAbout);
+
   /**
    * Creates and indexes a tombstone record.
    *
    * @param rdfAbout the id of the record
    * @return whether a record was tombstoned
    * @throws IndexingException in case something went wrong.
+   * @deprecated Use {@link #indexTombstone(String, DepublicationReason)}.
    */
+  //TODO: 2024-09-24 - Remove once ecloud has updated the code for tombstoning
+  @Deprecated(since = "13-SNAPSHOT", forRemoval = true)
   boolean indexTombstone(String rdfAbout) throws IndexingException;
 
+  /**
+   * Creates and indexes a tombstone record.
+   *
+   * @param rdfAbout the id of the record
+   * @param depublicationReason the depublication reason
+   * @return whether a record was tombstoned
+   * @throws IndexingException in case something went wrong.
+   */
+  boolean indexTombstone(String rdfAbout, DepublicationReason depublicationReason) throws IndexingException;
+
   /**
    * <p>
    * Removes all records that belong to a given dataset. This method also removes the associated

metis-indexing/src/main/java/eu/europeana/indexing/IndexerImpl.java

@@ -16,6 +16,7 @@
 import eu.europeana.indexing.tiers.model.TierResults;
 import eu.europeana.indexing.utils.RdfWrapper;
 import eu.europeana.metis.schema.jibx.RDF;
+import eu.europeana.metis.utils.DepublicationReason;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
@@ -136,11 +137,21 @@ public boolean remove(String rdfAbout) throws IndexerRelatedIndexingException {
   }
 
   @Override
-  public boolean indexTombstone(String rdfAbout) throws IndexerRelatedIndexingException {
+  public FullBeanImpl getTombstone(String rdfAbout){
+    return this.connectionProvider.getIndexedRecordAccess().getTombstoneFullbean(rdfAbout);
+  }
+
+  @Override
+  public boolean indexTombstone(String rdfAbout) throws IndexingException {
+    return indexTombstone(rdfAbout, DepublicationReason.GENERIC);
+  }
+
+  @Override
+  public boolean indexTombstone(String rdfAbout, DepublicationReason depublicationReason) throws IndexingException {
     final FullBeanImpl publishedFullbean = this.connectionProvider.getIndexedRecordAccess().getFullbean(rdfAbout);
     if (publishedFullbean != null) {
       final FullBeanPublisher publisher = connectionProvider.getFullBeanPublisher(true);
-      final FullBeanImpl tombstoneFullbean = prepareTombstoneFullbean(publishedFullbean);
+      final FullBeanImpl tombstoneFullbean = prepareTombstoneFullbean(publishedFullbean, depublicationReason);
       try {
         publisher.publishTombstone(tombstoneFullbean, tombstoneFullbean.getTimestampCreated());
       } catch (IndexingException e) {
@@ -150,24 +161,26 @@ public boolean indexTombstone(String rdfAbout) throws IndexerRelatedIndexingExce
     return publishedFullbean != null;
   }
 
-  private FullBeanImpl prepareTombstoneFullbean(FullBeanImpl publishedFullbean) {
+  private FullBeanImpl prepareTombstoneFullbean(FullBeanImpl publishedFullbean, DepublicationReason depublicationReason) {
     final FullBeanImpl tombstoneFullbean = new FullBeanImpl();
     tombstoneFullbean.setAbout(publishedFullbean.getAbout());
     tombstoneFullbean.setTimestampCreated(publishedFullbean.getTimestampCreated());
     tombstoneFullbean.setTimestampUpdated(publishedFullbean.getTimestampUpdated());
 
-    tombstoneFullbean.setEuropeanaAggregation(prepareEuropeanaAggregation(publishedFullbean.getEuropeanaAggregation()));
+    tombstoneFullbean.setEuropeanaAggregation(
+        prepareEuropeanaAggregation(publishedFullbean.getEuropeanaAggregation(), depublicationReason));
     tombstoneFullbean.setAggregations(List.of(prepareAggregation(publishedFullbean.getAggregations().getFirst())));
     final Optional<ProxyImpl> providerProxy =
         publishedFullbean.getProxies().stream().filter(not(ProxyImpl::isEuropeanaProxy)).findFirst();
     providerProxy.ifPresent(proxy -> tombstoneFullbean.setProxies(List.of(prepareProxy(proxy))));
     return tombstoneFullbean;
   }
 
-  private static EuropeanaAggregation prepareEuropeanaAggregation(EuropeanaAggregation europeanaAggregation) {
+  private static EuropeanaAggregation prepareEuropeanaAggregation(EuropeanaAggregation europeanaAggregation,
+      DepublicationReason depublicationReason) {
     final ChangeLog tombstoneChangeLog = new ChangeLogImpl();
     tombstoneChangeLog.setType("Delete");
-    tombstoneChangeLog.setContext("http://data.europeana.eu/vocabulary/depublicationReason/noPermission");
+    tombstoneChangeLog.setContext(depublicationReason.getUrl());
     tombstoneChangeLog.setEndTime(new Date());
     final EuropeanaAggregation tombstoneEuropeanaAggregation = new EuropeanaAggregationImpl();
     tombstoneEuropeanaAggregation.setAbout(europeanaAggregation.getAbout());

metis-indexing/src/main/java/eu/europeana/indexing/mongo/EuropeanaAggregationUpdater.java

@@ -29,5 +29,6 @@ protected void update(MongoPropertyUpdater<EuropeanaAggregationImpl> propertyUpd
     propertyUpdater.updateArray("aggregates", EuropeanaAggregation::getAggregates);
     propertyUpdater.updateWebResources("webResources", EuropeanaAggregation::getWebResources,
         ancestorInformation, new WebResourceUpdater());
+    propertyUpdater.updateObjectList("changeLog", EuropeanaAggregation::getChangeLog);
   }
 }

metis-indexing/src/test/java/eu/europeana/indexing/mongo/EuropeanaAggregationUpdaterTest.java

@@ -1,14 +1,22 @@
 package eu.europeana.indexing.mongo;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoMoreInteractions;
 
 import eu.europeana.corelib.definitions.edm.entity.EuropeanaAggregation;
 import eu.europeana.corelib.solr.entity.EuropeanaAggregationImpl;
 import eu.europeana.indexing.mongo.property.MongoPropertyUpdater;
 import eu.europeana.indexing.mongo.property.RootAboutWrapper;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Function;
 import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
 
 class EuropeanaAggregationUpdaterTest extends MongoEntityUpdaterTest<EuropeanaAggregationImpl> {
 
@@ -49,6 +57,15 @@ void testUpdate() {
     testWebResourcesPropertyUpdate(propertyUpdater, "webResources",
         EuropeanaAggregationImpl::setWebResources, rootAbout);
 
+    final EuropeanaAggregationImpl europeanaAggregation = new EuropeanaAggregationImpl();
+    europeanaAggregation.setChangeLog(new ArrayList<>());
+    @SuppressWarnings("unchecked")
+    final ArgumentCaptor<Function<EuropeanaAggregationImpl, List<Object>>> getterCaptor = ArgumentCaptor
+        .forClass(Function.class);
+    verify(propertyUpdater, times(1))
+        .updateObjectList(eq("changeLog"), getterCaptor.capture());
+    assertSame(europeanaAggregation.getChangeLog(), getterCaptor.getValue().apply(europeanaAggregation));
+
     // And that should be it.
     verifyNoMoreInteractions(propertyUpdater);
   }