Merge pull request kitodo#5961 from effective-webwork/mass-import-tests

Add tests for mass import

Kitodo/src/main/java/org/kitodo/constants/StringConstants.java

@@ -13,6 +13,7 @@
 
 public class StringConstants {
 
-    public static String COMMA_DELIMITER = ", ";
+    public static final String COMMA_DELIMITER = ", ";
+    public static final String SEMICOLON_DELIMITER = "; ";
 
 }

Kitodo/src/test/java/org/kitodo/production/forms/MassImportFormIT.java

@@ -0,0 +1,64 @@
+/*
+ * (c) Kitodo. Key to digital objects e. V. <[email protected]>
+ *
+ * This file is part of the Kitodo project.
+ *
+ * It is licensed under GNU General Public License version 3 or later.
+ *
+ * For the full copyright and license information, please read the
+ * GPL3-License.txt file that was distributed with this source code.
+ */
+
+package org.kitodo.production.forms;
+
+import java.util.List;
+
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.kitodo.MockDatabase;
+import org.kitodo.SecurityTestUtils;
+import org.kitodo.data.database.beans.User;
+import org.kitodo.production.forms.createprocess.ProcessDetail;
+import org.kitodo.production.forms.massimport.AddMetadataDialog;
+import org.kitodo.production.forms.massimport.MassImportForm;
+import org.kitodo.production.services.ServiceManager;
+
+/**
+ * Test for process mass import form.
+ */
+public class MassImportFormIT {
+
+    private static final int PROJECT_ID = 1;
+    private static final int TEMPLATE_ID = 1;
+    private static final String FIRST_TEMPLATE_TITLE = "First template";
+    private static final String TSL_ATS = "TSL/ATS";
+
+    @BeforeClass
+    public static void prepareDatabase() throws Exception {
+        MockDatabase.startNode();
+        MockDatabase.insertProcessesFull();
+        User userOne = ServiceManager.getUserService().getById(1);
+        SecurityTestUtils.addUserDataToSecurityContext(userOne, 1);
+    }
+
+    @AfterClass
+    public static void cleanup() throws Exception {
+        MockDatabase.stopNode();
+        MockDatabase.cleanDatabase();
+    }
+
+    @Test
+    public void shouldPrepareMassImport() {
+        MassImportForm massImportForm = new MassImportForm();
+        massImportForm.prepareMassImport(TEMPLATE_ID, PROJECT_ID);
+        Assert.assertEquals("Wrong template title", FIRST_TEMPLATE_TITLE, massImportForm.getTemplateTitle());
+        AddMetadataDialog addMetadataDialog = massImportForm.getAddMetadataDialog();
+        Assert.assertNotNull("'Add metadata' dialog should not be null", addMetadataDialog);
+        List<ProcessDetail> metadataTypes = addMetadataDialog.getAllMetadataTypes();
+        Assert.assertFalse("List of metadata types should not be empty", metadataTypes.isEmpty());
+        ProcessDetail firstDetail = metadataTypes.get(0);
+        Assert.assertEquals(String.format("First metadata type should be '%s'", TSL_ATS), TSL_ATS, firstDetail.getLabel());
+    }
+}

Kitodo/src/test/java/org/kitodo/production/services/catalogimport/MassImportTest.java

@@ -0,0 +1,103 @@
+/*
+ * (c) Kitodo. Key to digital objects e. V. <[email protected]>
+ *
+ * This file is part of the Kitodo project.
+ *
+ * It is licensed under GNU General Public License version 3 or later.
+ *
+ * For the full copyright and license information, please read the
+ * GPL3-License.txt file that was distributed with this source code.
+ */
+
+
+package org.kitodo.production.services.catalogimport;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.kitodo.constants.StringConstants;
+import org.kitodo.exceptions.ImportException;
+import org.kitodo.production.forms.CsvCell;
+import org.kitodo.production.forms.CsvRecord;
+import org.kitodo.production.services.ServiceManager;
+import org.kitodo.production.services.data.MassImportService;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+public class MassImportTest {
+
+    private static final String ID = "ID";
+    private static final String TITLE = "Title";
+    private static final String PLACE = "Place";
+    private static final List<String> METADATA_KEYS  = Arrays.asList(ID, TITLE, PLACE);
+    private static final String CSV_FIRST_LINE = "123, Band 1, Hamburg";
+    private static final String CSV_SECOND_LINE = "456, Band 2, Dresden";
+    private static final String CSV_THIRD_LINE = "789, Band 3, Berlin";
+    private static final List<String> CSV_LINES = Arrays.asList(CSV_FIRST_LINE, CSV_SECOND_LINE, CSV_THIRD_LINE);
+
+    /**
+     * Tests parsing CSV lines into CSV records with multiple cells.
+     */
+    @Test
+    public void shouldParseLines() {
+        MassImportService service = ServiceManager.getMassImportService();
+        // test parsing CSV lines with correct delimiter
+        List<CsvRecord> csvRecords = service.parseLines(CSV_LINES, StringConstants.COMMA_DELIMITER);
+        Assert.assertEquals("Wrong number of CSV records", 3, csvRecords.size());
+        List<CsvCell> cells = csvRecords.get(0).getCsvCells();
+        Assert.assertEquals("Wrong number of cells in first CSV record", 3, cells.size());
+        Assert.assertEquals("Wrong value in first cell of first CSV record", "123", cells.get(0).getValue());
+        Assert.assertEquals("Wrong value in second cell of first CSV record", "Band 1", cells.get(1).getValue());
+        Assert.assertEquals("Wrong value in third cell of first CSV record", "Hamburg", cells.get(2).getValue());
+        cells = csvRecords.get(1).getCsvCells();
+        Assert.assertEquals("Wrong number of cells in second CSV record", 3, cells.size());
+        Assert.assertEquals("Wrong value in first cell of second CSV record", "456", cells.get(0).getValue());
+        Assert.assertEquals("Wrong value in second cell of second CSV record", "Band 2", cells.get(1).getValue());
+        Assert.assertEquals("Wrong value in third cell of second CSV record", "Dresden", cells.get(2).getValue());
+        cells = csvRecords.get(2).getCsvCells();
+        Assert.assertEquals("Wrong number of cells in second CSV record", 3, cells.size());
+        Assert.assertEquals("Wrong value in first cell of third CSV record", "789", cells.get(0).getValue());
+        Assert.assertEquals("Wrong value in second cell of third CSV record", "Band 3", cells.get(1).getValue());
+        Assert.assertEquals("Wrong value in third cell of third CSV record", "Berlin", cells.get(2).getValue());
+        // test parsing CSV lines with incorrect delimiter
+        csvRecords = service.parseLines(CSV_LINES, ";");
+        cells = csvRecords.get(0).getCsvCells();
+        Assert.assertEquals("Wrong number of cells in first CSV record", 1, cells.size());
+        cells = csvRecords.get(1).getCsvCells();
+        Assert.assertEquals("Wrong number of cells in second CSV record", 1, cells.size());
+        cells = csvRecords.get(2).getCsvCells();
+        Assert.assertEquals("Wrong number of cells in third CSV record", 1, cells.size());
+    }
+
+    /**
+     * Tests whether updating CSV separator character from incorrect character to correct character keeps number of
+     * CSV records, but changes number of cells per record.
+     */
+    @Test
+    public void shouldUpdateSeparator() {
+        MassImportService service = ServiceManager.getMassImportService();
+        List<CsvRecord> oldCsvRecords = service.parseLines(CSV_LINES, StringConstants.SEMICOLON_DELIMITER);
+        Assert.assertTrue("CSV lines should not be separated into multiple records when using incorrect separator character",
+                oldCsvRecords.stream().noneMatch(csvRecord -> csvRecord.getCsvCells().size() > 1));
+        List<CsvRecord> newCsvRecords = service.updateSeparator(oldCsvRecords, StringConstants.SEMICOLON_DELIMITER, StringConstants.COMMA_DELIMITER);
+        Assert.assertEquals("Updating separator character should not alter number of CSV records", oldCsvRecords.size(), newCsvRecords.size());
+        Assert.assertTrue("CSV lines be separated into multiple records when using correct separator character",
+                newCsvRecords.stream().allMatch(csvRecord -> csvRecord.getCsvCells().size() > 1));
+    }
+
+    /**
+     * Tests whether parsing data entered in mass import form succeeds or not.
+     */
+    @Test
+    public void shouldPrepareMetadata() throws ImportException {
+        MassImportService service = ServiceManager.getMassImportService();
+        List<CsvRecord> csvRecords = service.parseLines(CSV_LINES, StringConstants.COMMA_DELIMITER);
+        Map<String, Map<String, String>> metadata = service.prepareMetadata(METADATA_KEYS, csvRecords);
+        Assert.assertEquals("Wrong number of metadata sets prepared", 3, metadata.size());
+        Map<String, String> metadataSet = metadata.get("123");
+        Assert.assertNotNull("Metadata for record with ID 123 is null", metadataSet);
+        Assert.assertTrue("Metadata for record with ID 123 does not contain title metadata", metadataSet.containsKey(TITLE));
+        Assert.assertEquals("Metadata for record with ID 123 contains wrong title", "Band 1", metadataSet.get(TITLE));
+    }
+}

Kitodo/src/test/java/org/kitodo/production/services/data/MassImportServiceIT.java

@@ -0,0 +1,103 @@
+/*
+ * (c) Kitodo. Key to digital objects e. V. <[email protected]>
+ *
+ * This file is part of the Kitodo project.
+ *
+ * It is licensed under GNU General Public License version 3 or later.
+ *
+ * For the full copyright and license information, please read the
+ * GPL3-License.txt file that was distributed with this source code.
+ */
+
+package org.kitodo.production.services.data;
+
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.kitodo.MockDatabase;
+import org.kitodo.api.Metadata;
+import org.kitodo.api.MetadataEntry;
+import org.kitodo.api.dataeditor.rulesetmanagement.RulesetManagementInterface;
+import org.kitodo.api.dataeditor.rulesetmanagement.StructuralElementViewInterface;
+import org.kitodo.data.database.beans.Ruleset;
+import org.kitodo.data.database.exceptions.DAOException;
+import org.kitodo.production.forms.createprocess.ProcessDetail;
+import org.kitodo.production.services.ServiceManager;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Locale;
+import java.util.stream.Collectors;
+
+public class MassImportServiceIT {
+
+    /**
+     * Prepare database before tests are run.
+     *
+     * @throws Exception when preparing database fails
+     */
+    @BeforeClass
+    public static void prepareDatabase() throws Exception {
+        MockDatabase.startNode();
+        MockDatabase.insertProcessesFull();
+    }
+
+    /**
+     * Cleanup database after all tests have been completed.
+     *
+     * @throws Exception when cleaning up database fails
+     */
+    @AfterClass
+    public static void cleanupDatabase() throws Exception {
+        MockDatabase.stopNode();
+        MockDatabase.cleanDatabase();
+    }
+
+    /**
+     * Tests whether preparing addable metadata depending on already entered metadata and divisions defined in ruleset
+     * works or not.
+     *
+     * @throws DAOException when retrieving list of divisions defined in test ruleset fails
+     * @throws IOException when loading test ruleset file fails
+     */
+    @Test
+    public void shouldGetAddableMetadataTable() throws DAOException, IOException {
+        Collection<Metadata> enteredMetadata = createPresetMetadata();
+        List<StructuralElementViewInterface> divisions = retrieveDivisions();
+        List<ProcessDetail> addableMetadata = ServiceManager.getMassImportService().getAddableMetadataTable(divisions, enteredMetadata);
+        Assert.assertFalse("List of addable metadata should not be empty",
+                addableMetadata.isEmpty());
+        Assert.assertTrue("List of addable metadata should contain 'TSL/ATS'",
+                addableMetadata.stream().anyMatch(m -> "TSL/ATS".equals(m.getLabel())));
+    }
+
+    private List<StructuralElementViewInterface> retrieveDivisions() throws DAOException, IOException {
+        Ruleset ruleset = ServiceManager.getRulesetService().getById(1);
+        RulesetManagementInterface rulesetInterface = ServiceManager.getRulesetService().openRuleset(ruleset);
+        List<Locale.LanguageRange> priorityList = Locale.LanguageRange.parse("en");
+        return rulesetInterface.getStructuralElements(priorityList).keySet().stream()
+                .map(key -> rulesetInterface.getStructuralElementView(key, "create", priorityList))
+                .collect(Collectors.toList());
+    }
+
+    private Collection<Metadata> createPresetMetadata() {
+        Collection<Metadata> presetMetadata = new LinkedList<>();
+        MetadataEntry titleMetadata = new MetadataEntry();
+        titleMetadata.setKey("title");
+        titleMetadata.setValue("Historische Zeitung");
+        presetMetadata.add(titleMetadata);
+        MetadataEntry placeMetadata = new MetadataEntry();
+        placeMetadata.setKey("place");
+        placeMetadata.setValue("Hamburg");
+        presetMetadata.add(placeMetadata);
+        MetadataEntry authorMetadata = new MetadataEntry();
+        authorMetadata.setKey("author");
+        authorMetadata.setValue("Hans Meier");
+        presetMetadata.add(authorMetadata);
+        return presetMetadata;
+    }
+
+}

Kitodo/src/test/java/org/kitodo/selenium/MassImportST.java

@@ -0,0 +1,112 @@
+/*
+ * (c) Kitodo. Key to digital objects e. V. <[email protected]>
+ *
+ * This file is part of the Kitodo project.
+ *
+ * It is licensed under GNU General Public License version 3 or later.
+ *
+ * For the full copyright and license information, please read the
+ * GPL3-License.txt file that was distributed with this source code.
+ */
+
+package org.kitodo.selenium;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.kitodo.MockDatabase;
+import org.kitodo.constants.StringConstants;
+import org.kitodo.selenium.testframework.BaseTestSelenium;
+import org.kitodo.selenium.testframework.Browser;
+import org.kitodo.selenium.testframework.Pages;
+import org.kitodo.selenium.testframework.pages.MassImportPage;
+import org.openqa.selenium.By;
+import org.openqa.selenium.WebElement;
+
+public class MassImportST extends BaseTestSelenium {
+
+    private static final String ID = "ID";
+    private static final String TITLE = "Title";
+    private static final String PLACE = "Place";
+    private static final List<String> METADATA_KEYS  = Arrays.asList(ID, TITLE, PLACE);
+    private static File csvUploadFile;
+    private static MassImportPage massImportPage;
+    private static final String CSV_UPLOAD_FILENAME = "test_import";
+    private static final String CSV_UPLOAD_FILE_EXTENSION = ".csv";
+    private static final String CSV_CELL_SELECTOR = "#editForm\\:recordsTable_data tr .ui-cell-editor-output";
+
+    @BeforeClass
+    public static void setup() throws Exception {
+        massImportPage = Pages.getMassImportPage();
+        csvUploadFile = createCsvFile();
+        MockDatabase.insertMappingFiles();
+        MockDatabase.insertImportConfigurations();
+    }
+
+    @Before
+    public void login() throws Exception {
+        Pages.getLoginPage().goTo().performLoginAsAdmin();
+        Pages.getProjectsPage().goTo();
+        Pages.getProjectsPage().clickMassImportAction();
+    }
+
+    @After
+    public void logout() throws Exception {
+        Pages.getTopNavigation().logout();
+        if (Browser.isAlertPresent()) {
+            Browser.getDriver().switchTo().alert().accept();
+        }
+    }
+
+    @AfterClass
+    public static void cleanup() throws IOException {
+        deleteCsvFile();
+    }
+
+    /**
+     * Tests whether uploaded CSV files in mass import form are parsed correctly or not.
+     */
+    @Test
+    public void handleCsvFileUpload() throws InterruptedException {
+        massImportPage.uploadTestCsvFile(csvUploadFile.getAbsolutePath());
+        Thread.sleep(Browser.getDelayAfterLogout());
+        List<WebElement> csvRows = Browser.getDriver().findElement(By.id("editForm:recordsTable_data"))
+                .findElements(By.tagName("tr"));
+        Assert.assertEquals("CSV file not parsed correctly", 3, csvRows.size());
+        List<WebElement> csvCells = Browser.getDriver().findElements(By.cssSelector(CSV_CELL_SELECTOR));
+        Assert.assertEquals("CSV lines should not be segmented correctly into multiple cells when using wrong CSV "
+                + "separator", 3, csvCells.size());
+        massImportPage.updateSeparator(StringConstants.COMMA_DELIMITER.trim());
+        List<WebElement> updatedCsvCells = Browser.getDriver().findElements(By.cssSelector(CSV_CELL_SELECTOR));
+        Assert.assertEquals("CSV lines should be segmented correctly into multiple cells when using correct CSV "
+                + "separator", 9, updatedCsvCells.size());
+    }
+
+    private static File createCsvFile() throws IOException {
+        File csvFile = File.createTempFile(CSV_UPLOAD_FILENAME, CSV_UPLOAD_FILE_EXTENSION);
+        try (FileWriter writer = new FileWriter(csvFile)) {
+            writer.write(String.join(StringConstants.COMMA_DELIMITER, METADATA_KEYS) + "\n");
+            writer.write("123, Band 1, Hamburg\n");
+            writer.write("456, Band 2, Dresden\n");
+            writer.write("789, Band 3, Berlin");
+        }
+        return csvFile;
+    }
+
+    private static void deleteCsvFile() throws IOException {
+        boolean successfullyDeleted = csvUploadFile.delete();
+        if (!successfullyDeleted) {
+            throw new IOException(String.format("Error deleting CSV test file '%s'", csvUploadFile.getName()));
+        }
+    }
+
+}