Merge pull request #13 from europeana/EA-3676_multipleRetriveal

Ea 3676 multiple retriveal
europeana · Mar 22, 2024 · 39cf8d3 · 39cf8d3
2 parents 42f6c08 + 4f6e508
commit 39cf8d3
Show file tree

Hide file tree

Showing 8 changed files with 46 additions and 54 deletions.
diff --git a/record-api-common/src/main/java/eu/europeana/api/format/FormatWriter.java b/record-api-common/src/main/java/eu/europeana/api/format/FormatWriter.java
@@ -2,16 +2,15 @@
 
 import java.io.IOException;
 import java.io.OutputStream;
-import java.util.List;
+import java.util.Iterator;
 
 /**
  * @author Hugo
  * @since 12 Oct 2023
  */
-public interface FormatWriter<T>
-{
+public interface FormatWriter<T> {
     public void write(T value, OutputStream out) throws IOException;
 
-    public void write(List<T> value, OutputStream out) throws IOException;
+    public void write(Iterator<T> value, int size, OutputStream out) throws IOException;
 
 }
diff --git a/record-api-model/src/main/java/eu/europeana/api/record/io/JenaBasedFormatWriter.java b/record-api-model/src/main/java/eu/europeana/api/record/io/JenaBasedFormatWriter.java
@@ -10,7 +10,7 @@
 import javax.annotation.Resource;
 import java.io.IOException;
 import java.io.OutputStream;
-import java.util.List;
+import java.util.Iterator;
 
 /**
  * @author Hugo
@@ -35,7 +35,7 @@ public void write(ProvidedCHO cho, OutputStream out) throws IOException {
     }
 
     @Override
-    public void write(List<ProvidedCHO> value, OutputStream out) throws IOException {
+    public void write(Iterator<ProvidedCHO> value, int size, OutputStream out) throws IOException {
         // empty for now
     }
 }
diff --git a/record-api-model/src/main/java/eu/europeana/api/record/io/json/JsonLdWriter.java b/record-api-model/src/main/java/eu/europeana/api/record/io/json/JsonLdWriter.java
@@ -2,17 +2,16 @@
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.cfg.ContextAttributes;
-import com.fasterxml.jackson.databind.node.ArrayNode;
-import com.fasterxml.jackson.databind.node.ObjectNode;
 import eu.europeana.api.config.AppConfigConstants;
 import eu.europeana.api.format.FormatWriter;
 import eu.europeana.api.record.model.ProvidedCHO;
 import org.springframework.stereotype.Component;
 
 import javax.annotation.Resource;
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
-import java.util.List;
+import java.util.Iterator;
 import java.util.Stack;
 
 import static eu.europeana.api.config.AppConfigConstants.BEAN_JSON_MAPPER;
@@ -50,29 +49,36 @@ public void write(ProvidedCHO providedCHO, OutputStream out) throws IOException
     }
 
     @Override
-    public void write(List<ProvidedCHO> providedCHOS, OutputStream out) throws IOException {
+    public void write(Iterator<ProvidedCHO> providedCHOS, int size, OutputStream out) throws IOException {
         Stack lStack = new Stack<String>();
         stack.set(lStack);
-        ArrayNode records = mapper.createArrayNode();
-        providedCHOS.stream()
-                .forEach(
-                        providedCHO -> {
-                            try {
-                                ContextAttributes attrs = ContextAttributes.getEmpty().withSharedAttribute(context, new Context(providedCHO.getID()));
-                                mapper.setDefaultAttributes(attrs);
-                                records.add(mapper.valueToTree(providedCHO));
-                            } finally {
-                                lStack.clear();
-                            }
-                        });
-
-        ObjectNode result = mapper.createObjectNode();
-        result.set("type", mapper.valueToTree("ResultPage"));
-        result.set("total", mapper.valueToTree(records.size()));
-        result.set("items", records);
+        ByteArrayOutputStream stream = new ByteArrayOutputStream();
+        stream.write("{\"type\": \"ResultPage\",".getBytes());
+        stream.write(("\"total\":" + size + ",").getBytes());
+        stream.write(("\"items\": [" ).getBytes());
+        int counter = 1;
+        while (providedCHOS.hasNext()) {
+            ProvidedCHO providedCHO = providedCHOS.next();
+            ContextAttributes attrs = ContextAttributes.getEmpty()
+                    .withSharedAttribute(context, new Context(providedCHO.getID()));
+            mapper.setDefaultAttributes(attrs);
+            try {
+                mapper.writerWithDefaultPrettyPrinter().writeValues(stream).write(providedCHO);
+                if (counter < size) {
+                    stream.write(",".getBytes());
+                }
+                counter ++;
+            }
+            finally {
+                lStack.clear();
+            }
+        }
 
+        stream.write("]}".getBytes());
+        stream.flush();
+        stream.writeTo(out);
         stack.remove();
-        mapper.writerWithDefaultPrettyPrinter().writeValues(out).write(result);
+        stream.close();
     }
 }
 
diff --git a/record-api-model/src/main/java/eu/europeana/api/record/io/json/v2/JsonV2Writer.java b/record-api-model/src/main/java/eu/europeana/api/record/io/json/v2/JsonV2Writer.java
@@ -5,6 +5,7 @@
 
 import java.io.IOException;
 import java.io.OutputStream;
+import java.util.Iterator;
 import java.util.List;
 
 import eu.europeana.api.config.AppConfigConstants;
@@ -40,7 +41,7 @@ public void write(ProvidedCHO cho, OutputStream out) throws IOException {
     }
 
     @Override
-    public void write(List<ProvidedCHO> value, OutputStream out) throws IOException {
+    public void write(Iterator<ProvidedCHO> value, int size, OutputStream out) throws IOException {
         // empty for now
     }
 

diff --git a/record-api-model/src/main/java/eu/europeana/api/record/io/xml/XmlRecordWriter.java b/record-api-model/src/main/java/eu/europeana/api/record/io/xml/XmlRecordWriter.java
@@ -12,7 +12,7 @@
 import javax.xml.stream.XMLStreamException;
 import java.io.IOException;
 import java.io.OutputStream;
-import java.util.List;
+import java.util.Iterator;
 
 /**
  * @author Hugo
@@ -40,7 +40,7 @@ public void write(ProvidedCHO cho, OutputStream out) throws IOException {
     }
 
     @Override
-    public void write(List<ProvidedCHO> value, OutputStream out) throws IOException {
+    public void write(Iterator<ProvidedCHO> value, int size, OutputStream out) throws IOException {
         // empty for now
     }
 }
diff --git a/record-api-mongo/src/main/java/eu/europeana/api/record/db/repository/RecordRepository.java b/record-api-mongo/src/main/java/eu/europeana/api/record/db/repository/RecordRepository.java
@@ -2,6 +2,7 @@
 
 import dev.morphia.Datastore;
 import dev.morphia.query.FindOptions;
+import dev.morphia.query.MorphiaCursor;
 import dev.morphia.query.filters.Filter;
 import dev.morphia.query.filters.Filters;
 import eu.europeana.api.config.AppConfigConstants;
@@ -76,14 +77,13 @@ public ProvidedCHO findById(String about) {
      * @param recordIds ids to be fetched
      * @return list of ProvidedCho(s)
      */
-    public List<ProvidedCHO> findByRecordIds(List<String> recordIds) {
+    public MorphiaCursor<ProvidedCHO> findByRecordIds(List<String> recordIds) {
         List<Filter> filters = new ArrayList<>();
         filters.add(Filters.in("id", recordIds));
         return datastore
                 .find(ProvidedCHO.class)
                 .filter(filters.toArray(Filter[]::new))
-                .iterator(new FindOptions())
-                .toList();
+                .iterator(new FindOptions());
     }
 
 

diff --git a/record-api-web/src/main/java/eu/europeana/api/record/service/RecordService.java b/record-api-web/src/main/java/eu/europeana/api/record/service/RecordService.java
@@ -1,5 +1,6 @@
 package eu.europeana.api.record.service;
 
+import dev.morphia.query.MorphiaCursor;
 import eu.europeana.api.record.config.AppConfig;
 import eu.europeana.api.record.config.RecordApiConfiguration;
 import eu.europeana.api.record.model.ProvidedCHO;
@@ -31,7 +32,7 @@ public Optional<ProvidedCHO> getRecord(String about) {
         return Optional.ofNullable(recordRepository.findById(about));
     }
 
-    public List<ProvidedCHO> retrieveMultipleByRecordIds(List<String> recordIds) {
+    public MorphiaCursor<ProvidedCHO> retrieveMultipleByRecordIds(List<String> recordIds) {
         return recordRepository.findByRecordIds(recordIds);
     }
 

diff --git a/record-api-web/src/main/java/eu/europeana/api/record/web/RecordController.java b/record-api-web/src/main/java/eu/europeana/api/record/web/RecordController.java
@@ -1,5 +1,6 @@
 package eu.europeana.api.record.web;
 
+import dev.morphia.query.MorphiaCursor;
 import eu.europeana.api.commons.web.http.HttpHeaders;
 import eu.europeana.api.error.EuropeanaApiException;
 import eu.europeana.api.format.RdfFormat;
@@ -23,7 +24,6 @@
 
 import java.io.*;
 import java.util.*;
-import java.util.stream.Collectors;
 
 import static eu.europeana.api.record.utils.RecordConstants.*;
 
@@ -118,33 +118,18 @@ public void writeTo(OutputStream out) throws IOException {
 
     private ResponseEntity<StreamingResponseBody> createResponseMultipleRecords(List<String> urls) throws EuropeanaApiException {
         List<String> recordIds = RecordUtils.buildRecordIds(urls);
-        List<ProvidedCHO> records = recordService.retrieveMultipleByRecordIds(recordIds);
-        if (records.isEmpty()) {
+        MorphiaCursor<ProvidedCHO> records = recordService.retrieveMultipleByRecordIds(recordIds);
+        if (records.available() == 0) {
             throw new RecordDoesNotExistsException(urls.toString());
         }
 
-        // LinkedHashMap iterates keys() and values() in order of insertion. Using a map
-        // improves sort performance significantly
-        Map<String, ProvidedCHO> sortedRecordMap = new LinkedHashMap<>(records.size());
-        for (String id : recordIds) {
-            sortedRecordMap.put(id, null);
-        }
-        for (ProvidedCHO providedCHO : records) {
-            sortedRecordMap.replace(providedCHO.getID(), providedCHO);
-        }
-
         // create response headers
         org.springframework.http.HttpHeaders httpHeaders= new org.springframework.http.HttpHeaders();
         httpHeaders.setContentType(MediaType.valueOf(RdfFormat.JSONLD.getMediaType()));
-
-        // remove null values in response
-        List<ProvidedCHO> providedCHOS = sortedRecordMap.values().stream()
-                .filter(Objects::nonNull).collect(Collectors.toList());
-
         StreamingResponseBody responseBody = new StreamingResponseBody() {
             @Override
             public void writeTo(OutputStream out) throws IOException {
-                formatHandlerRegistry.get(RdfFormat.JSONLD).write(providedCHOS, out);
+                formatHandlerRegistry.get(RdfFormat.JSONLD).write(records, records.available(), out);
                 out.flush();
             }
         };