Replaces CR and LF characters with a safe value e.g. ""(empty string).
+ * + * @param input the input + * @return the sanitized input, safe for logging + */ + public static String sanitizeCRLF(String input) { + return input == null ? null : CRLF_PATTERN.matcher(input).replaceAll(""); + } } diff --git a/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/DepublicationReason.java b/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/DepublicationReason.java index fad683529..444a440ac 100644 --- a/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/DepublicationReason.java +++ b/metis-common/metis-common-utils/src/main/java/eu/europeana/metis/utils/DepublicationReason.java @@ -2,28 +2,39 @@ /** * Enum for depublication reason. + *Note: The enum value {@link #UNKNOWN} is to be used for historical depublication workflows(before the reason was + * implemented). + * In other words the historical workflows will be populated by a script once with the {@link #UNKNOWN} reason, and this value + * should never be used during depublication since its release. Therefore the url is an empty string and not meant to be used for + * populating records in the database(e.g. tombstoning)
*/ public enum DepublicationReason { - BROKEN_MEDIA_LINKS("Broken media links", "http://data.europeana.eu/vocabulary/depublicationReason/contentTier0"), - GDPR("GDPR", "http://data.europeana.eu/vocabulary/depublicationReason/gdpr"), - PERMISSION_ISSUES("Permission issues", "http://data.europeana.eu/vocabulary/depublicationReason/noPermission"), - SENSITIVE_CONTENT("Sensitive content", "http://data.europeana.eu/vocabulary/depublicationReason/sensitiveContent"), - REMOVED_DATA_AT_SOURCE("Removed data at source", "http://data.europeana.eu/vocabulary/depublicationReason/sourceRemoval"), - GENERIC("Generic", "http://data.europeana.eu/vocabulary/depublicationReason/generic"), - UNKNOWN("Unknown", "http://data.europeana.eu/vocabulary/depublicationReason/unknown"); + BROKEN_MEDIA_LINKS("Broken media links", "contentTier0"), + GDPR("GDPR", "gdpr"), + PERMISSION_ISSUES("Permission issues", "noPermission"), + SENSITIVE_CONTENT("Sensitive content", "sensitiveContent"), + REMOVED_DATA_AT_SOURCE("Removed data at source", "sourceRemoval"), + GENERIC("Generic", "generic"), + UNKNOWN("Unknown", ""); - private final String valueAsString; + private static final String BASE_URL = "http://data.europeana.eu/vocabulary/depublicationReason/"; + + private final String title; private final String url; - DepublicationReason(String valueAsString, String url) { - this.valueAsString = valueAsString; - this.url = url; + DepublicationReason(String title, String urlSuffix) { + this.title = title; + this.url = BASE_URL + urlSuffix; } @Override - public String toString(){ - return valueAsString; + public String toString() { + return title; + } + + public String getTitle() { + return title; } public String getUrl() { diff --git a/metis-common/metis-common-utils/src/test/java/eu/europeana/metis/utils/CommonStringValuesTest.java b/metis-common/metis-common-utils/src/test/java/eu/europeana/metis/utils/CommonStringValuesTest.java new file mode 100644 index 000000000..5c0433414 --- /dev/null +++ b/metis-common/metis-common-utils/src/test/java/eu/europeana/metis/utils/CommonStringValuesTest.java @@ -0,0 +1,88 @@ +package eu.europeana.metis.utils; + +import static eu.europeana.metis.utils.CommonStringValues.BATCH_OF_DATASETS_RETURNED; +import static eu.europeana.metis.utils.CommonStringValues.CRLF_PATTERN; +import static eu.europeana.metis.utils.CommonStringValues.DATE_FORMAT; +import static eu.europeana.metis.utils.CommonStringValues.DATE_FORMAT_FOR_REQUEST_PARAM; +import static eu.europeana.metis.utils.CommonStringValues.DATE_FORMAT_FOR_SCHEDULING; +import static eu.europeana.metis.utils.CommonStringValues.DATE_FORMAT_Z; +import static eu.europeana.metis.utils.CommonStringValues.EUROPEANA_ID_CREATOR_INITIALIZATION_FAILED; +import static eu.europeana.metis.utils.CommonStringValues.NEXT_PAGE_CANNOT_BE_NEGATIVE; +import static eu.europeana.metis.utils.CommonStringValues.PAGE_COUNT_CANNOT_BE_ZERO_OR_NEGATIVE; +import static eu.europeana.metis.utils.CommonStringValues.PLUGIN_EXECUTION_NOT_ALLOWED; +import static eu.europeana.metis.utils.CommonStringValues.REPLACEABLE_CRLF_CHARACTERS_REGEX; +import static eu.europeana.metis.utils.CommonStringValues.S_DATA_PROVIDERS_S_DATA_SETS_S_TEMPLATE; +import static eu.europeana.metis.utils.CommonStringValues.UNAUTHORIZED; +import static eu.europeana.metis.utils.CommonStringValues.WRONG_ACCESS_TOKEN; +import static eu.europeana.metis.utils.CommonStringValues.sanitizeCRLF; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +import java.util.regex.Pattern; +import org.junit.jupiter.api.Test; + +class CommonStringValuesTest { + + @Test + void testFieldsAreUsed() { + assertNotNull(WRONG_ACCESS_TOKEN); + assertNotNull(BATCH_OF_DATASETS_RETURNED); + assertNotNull(NEXT_PAGE_CANNOT_BE_NEGATIVE); + assertNotNull(PAGE_COUNT_CANNOT_BE_ZERO_OR_NEGATIVE); + assertNotNull(PLUGIN_EXECUTION_NOT_ALLOWED); + assertNotNull(UNAUTHORIZED); + assertNotNull(EUROPEANA_ID_CREATOR_INITIALIZATION_FAILED); + assertNotNull(DATE_FORMAT); + assertNotNull(DATE_FORMAT_Z); + assertNotNull(DATE_FORMAT_FOR_SCHEDULING); + assertNotNull(DATE_FORMAT_FOR_REQUEST_PARAM); + assertNotNull(S_DATA_PROVIDERS_S_DATA_SETS_S_TEMPLATE); + assertNotNull(REPLACEABLE_CRLF_CHARACTERS_REGEX); + assertNotNull(CRLF_PATTERN); + } + + @Test + void testPattern() { + Pattern expectedPattern = Pattern.compile("[\r\n\t]"); + assertEquals(expectedPattern.pattern(), CRLF_PATTERN.pattern()); + } + + @Test + void testSanitizeCRLF_NullInput() { + assertNull(sanitizeCRLF(null)); + } + + @Test + void testSanitizeStringForLogging_EmptyString() { + String input = ""; + assertEquals("", sanitizeCRLF(input)); + } + + @Test + void testSanitizeCRLF_NoSpecialCharacters() { + String input = "This is a test."; + assertEquals("This is a test.", sanitizeCRLF(input)); + } + + @Test + void testSanitizeCRLF_WithCRLFCharacters() { + String input = "This is a test.\nThis is a new line.\rThis is a carriage return.\tThis is a tab."; + String expected = "This is a test.This is a new line.This is a carriage return.This is a tab."; + assertEquals(expected, sanitizeCRLF(input)); + } + + @Test + void testSanitizeCRLF_MixedInput() { + String input = "\r\n\tThis string has special characters at the start.\r\n"; + String expected = "This string has special characters at the start."; + assertEquals(expected, sanitizeCRLF(input)); + } + + @Test + void testSanitizeCRLF_NoCRLFCharacters() { + String input = "Regular string without CRLF."; + assertEquals("Regular string without CRLF.", sanitizeCRLF(input)); + } +} + diff --git a/metis-common/metis-common-utils/src/test/java/eu/europeana/metis/utils/DepublicationReasonTest.java b/metis-common/metis-common-utils/src/test/java/eu/europeana/metis/utils/DepublicationReasonTest.java new file mode 100644 index 000000000..7d5bc7a4d --- /dev/null +++ b/metis-common/metis-common-utils/src/test/java/eu/europeana/metis/utils/DepublicationReasonTest.java @@ -0,0 +1,54 @@ +package eu.europeana.metis.utils; + +import static eu.europeana.metis.utils.DepublicationReason.BROKEN_MEDIA_LINKS; +import static eu.europeana.metis.utils.DepublicationReason.GDPR; +import static eu.europeana.metis.utils.DepublicationReason.GENERIC; +import static eu.europeana.metis.utils.DepublicationReason.PERMISSION_ISSUES; +import static eu.europeana.metis.utils.DepublicationReason.REMOVED_DATA_AT_SOURCE; +import static eu.europeana.metis.utils.DepublicationReason.SENSITIVE_CONTENT; +import static eu.europeana.metis.utils.DepublicationReason.UNKNOWN; +import static eu.europeana.metis.utils.DepublicationReason.values; +import static java.util.Arrays.asList; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Arrays; +import java.util.List; +import org.junit.jupiter.api.Test; + +class DepublicationReasonTest { + + @Test + void testValues() { + Arrays.stream(values()).forEach(depublicationReason -> { + assertNotNull(depublicationReason.getTitle()); + assertNotNull(depublicationReason.getUrl()); + }); + } + + @Test + void testToStringMethod() { + assertEquals("Broken media links", BROKEN_MEDIA_LINKS.toString()); + assertEquals("GDPR", GDPR.toString()); + assertEquals("Permission issues", PERMISSION_ISSUES.toString()); + assertEquals("Sensitive content", SENSITIVE_CONTENT.toString()); + assertEquals("Removed data at source", REMOVED_DATA_AT_SOURCE.toString()); + assertEquals("Generic", GENERIC.toString()); + assertEquals("Unknown", UNKNOWN.toString()); + } + + @Test + void testEnumValuePresence() { + List- * This method indexes a single record, publishing it to the provided data stores. + * This method indexes a single rdf, publishing it to the provided data stores. *
** NOTE: this operation should not coincide with a remove operation as this operation is * not done within a transaction. *
* - * @param record The record to index. + * @param rdf The rdf to index. * @param indexingProperties The properties of this indexing operation. * @throws IndexingException In case a problem occurred during indexing. */ - void indexRdf(RDF record, IndexingProperties indexingProperties) throws IndexingException; + void indexRdf(RDF rdf, IndexingProperties indexingProperties) throws IndexingException; /** *@@ -60,18 +60,18 @@ public interface Indexer extends Closeable { /** *
- * This method indexes a single record, publishing it to the provided data stores. + * This method indexes a single rdfString, publishing it to the provided data stores. *
** NOTE: this operation should not coincide with a remove operation as this operation is * not done within a transaction. *
* - * @param record The record to index (can be parsed to RDF). + * @param rdfString The rdfString to index (can be parsed to RDF). * @param indexingProperties The properties of this indexing operation. * @throws IndexingException In case a problem occurred during indexing. */ - void index(String record, IndexingProperties indexingProperties) throws IndexingException; + void index(String rdfString, IndexingProperties indexingProperties) throws IndexingException; /** *@@ -110,18 +110,18 @@ void index(String stringRdfRecord, IndexingProperties indexingProperties, /** *
- * This method indexes a single record, publishing it to the provided data stores. + * This method indexes a single rdfInputStream, publishing it to the provided data stores. *
** NOTE: this operation should not coincide with a remove operation as this operation is * not done within a transaction. *
* - * @param record The record to index (can be parsed to RDF). + * @param rdfInputStream The rdfInputStream to index (can be parsed to RDF). * @param indexingProperties The properties of this indexing operation. * @throws IndexingException In case a problem occurred during indexing. */ - void index(InputStream record, IndexingProperties indexingProperties) throws IndexingException; + void index(InputStream rdfInputStream, IndexingProperties indexingProperties) throws IndexingException; /** *
@@ -188,18 +188,6 @@ TierResults indexAndGetTierCalculations(InputStream recordContent,
*/
FullBeanImpl getTombstone(String rdfAbout);
- /**
- * Creates and indexes a tombstone record.
- *
- * @param rdfAbout the id of the record
- * @return whether a record was tombstoned
- * @throws IndexingException in case something went wrong.
- * @deprecated Use {@link #indexTombstone(String, DepublicationReason)}.
- */
- //TODO: 2024-09-24 - Remove once ecloud has updated the code for tombstoning
- @Deprecated(since = "13-SNAPSHOT", forRemoval = true)
- boolean indexTombstone(String rdfAbout) throws IndexingException;
-
/**
* Creates and indexes a tombstone record.
*
diff --git a/metis-indexing/src/main/java/eu/europeana/indexing/IndexerImpl.java b/metis-indexing/src/main/java/eu/europeana/indexing/IndexerImpl.java
index 5e1f6d608..64a8df7d8 100644
--- a/metis-indexing/src/main/java/eu/europeana/indexing/IndexerImpl.java
+++ b/metis-indexing/src/main/java/eu/europeana/indexing/IndexerImpl.java
@@ -1,5 +1,6 @@
package eu.europeana.indexing;
+import static java.lang.String.format;
import static java.util.function.Predicate.not;
import eu.europeana.corelib.definitions.edm.entity.ChangeLog;
@@ -84,10 +85,10 @@ public void index(List