Merge pull request #143 from aodn/features/5905-optimize-load-speed

Speed up map processing by using multiple thread

indexer/src/main/java/au/org/aodn/esindexer/configuration/IndexerConfig.java

@@ -14,6 +14,7 @@
 import org.springframework.beans.factory.annotation.Value;
 
 import javax.annotation.PostConstruct;
+import javax.annotation.PreDestroy;
 
 @Configuration
 @EnableRetry
@@ -25,6 +26,13 @@ public class IndexerConfig {
  @PostConstruct
  public void init() {
  GeometryUtils.setCellSize(cellSize);
+ GeometryUtils.setExecutorService(Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()));
+ }
+
+ @PreDestroy
+ public void cleanUp() {
+ // Clean up resources
+ GeometryUtils.getExecutorService().shutdown();
  }
  /**
  * We need to create component here because we do not want to run test with real http connection

indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java

@@ -23,6 +23,7 @@
 import java.math.RoundingMode;
 import java.net.URL;
 import java.util.*;
+import java.util.concurrent.*;
 import java.util.function.Function;
 
 public class GeometryUtils {
@@ -31,6 +32,10 @@ public class GeometryUtils {
  protected static GeometryFactory factory = new GeometryFactory(new PrecisionModel(), 4326);
  protected static ObjectMapper objectMapper = new ObjectMapper();
  protected static Geometry landGeometry;
+ // Create an ExecutorService with a fixed thread pool size
+ @Getter
+ @Setter
+ protected static ExecutorService executorService;
 
  @Getter
  @Setter
@@ -271,22 +276,45 @@ protected static List<Geometry> convertToListGeometry(Geometry multipolygon) {
  * @param large - A Polygon to break into grid
  * @return - A polygon the break into grid.
  */
- protected static List<Geometry> breakLargeGeometryToGrid(Geometry large) {
+ protected static List<Geometry> breakLargeGeometryToGrid(final Geometry large) {
+ logger.debug("Start break down large geometry");
  // Get the bounding box (extent) of the large polygon
  Envelope envelope = large.getEnvelopeInternal();
 
  // Hard code cell size, we can adjust the break grid size. 10.0 result in 3x3 grid
  // cover Australia
  List<Polygon> gridPolygons = createGridPolygons(envelope, getCellSize());
 
- List<Geometry> intersectedPolygons = new ArrayList<>();
+ // List to store Future objects representing the results of the tasks
+ List<Future<Geometry>> futureResults = new ArrayList<>();
+
+ // Submit tasks to executor for each gridPolygon
  for (Polygon gridPolygon : gridPolygons) {
- Geometry intersection = gridPolygon.intersection(large);
- if (!intersection.isEmpty()) {
- intersectedPolygons.add(intersection);
+ Callable<Geometry> task = () -> {
+ Geometry intersection = gridPolygon.intersection(large);
+ return !intersection.isEmpty() ? intersection : null;
+ };
+ Future<Geometry> future = executorService.submit(task);
+ futureResults.add(future);
+ }
+
+ // List to store the intersected polygons
+ final List<Geometry> intersectedPolygons = new ArrayList<>();
+
+ // Collect the results from the futures
+ for (Future<Geometry> future : futureResults) {
+ try {
+ // This blocks until the result is available
+ Geometry result = future.get();
+ if (result != null) {
+ intersectedPolygons.add(result);
+ }
+ } catch (InterruptedException | ExecutionException e) {
+ // Nothing to report
  }
  }
 
+ logger.debug("End break down large geometry");
  return intersectedPolygons;
  }
 

indexer/src/main/resources/application-edge.yaml

@@ -9,3 +9,7 @@ management:
  web:
  exposure:
  include: "health,info,env,beans,logfile"
+
+logging:
+ level:
+ au.org.aodn.ardcvocabs: DEBUG