From 4c41a2fc1f3d2664efd1de428c1c1b72b8e18fa8 Mon Sep 17 00:00:00 2001 From: utas-raymondng Date: Tue, 24 Sep 2024 18:05:05 +1000 Subject: [PATCH 1/4] Speed up map processing by using multiple thread --- .../aodn/esindexer/utils/GeometryUtils.java | 40 ++++++++++++++++--- .../src/main/resources/application-edge.yaml | 4 ++ 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java b/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java index 1a48344d..74c49650 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java @@ -23,6 +23,7 @@ import java.math.RoundingMode; import java.net.URL; import java.util.*; +import java.util.concurrent.*; import java.util.function.Function; public class GeometryUtils { @@ -271,7 +272,8 @@ protected static List convertToListGeometry(Geometry multipolygon) { * @param large - A Polygon to break into grid * @return - A polygon the break into grid. */ - protected static List breakLargeGeometryToGrid(Geometry large) { + protected static List breakLargeGeometryToGrid(final Geometry large) { + logger.debug("Start break down large geometry"); // Get the bounding box (extent) of the large polygon Envelope envelope = large.getEnvelopeInternal(); @@ -279,14 +281,42 @@ protected static List breakLargeGeometryToGrid(Geometry large) { // cover Australia List gridPolygons = createGridPolygons(envelope, getCellSize()); - List intersectedPolygons = new ArrayList<>(); + // Create an ExecutorService with a fixed thread pool size + ExecutorService executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); + + // List to store Future objects representing the results of the tasks + List> futureResults = new ArrayList<>(); + + // Submit tasks to executor for each gridPolygon for (Polygon gridPolygon : gridPolygons) { - Geometry intersection = gridPolygon.intersection(large); - if (!intersection.isEmpty()) { - intersectedPolygons.add(intersection); + Callable task = () -> { + Geometry intersection = gridPolygon.intersection(large); + return !intersection.isEmpty() ? intersection : null; + }; + Future future = executorService.submit(task); + futureResults.add(future); + } + + // List to store the intersected polygons + final List intersectedPolygons = new ArrayList<>(); + + // Collect the results from the futures + for (Future future : futureResults) { + try { + // This blocks until the result is available + Geometry result = future.get(); + if (result != null) { + intersectedPolygons.add(result); + } + } catch (InterruptedException | ExecutionException e) { + // Nothing to report } } + // Shutdown the ExecutorService after all tasks are completed + executorService.shutdown(); + + logger.debug("End break down large geometry"); return intersectedPolygons; } diff --git a/indexer/src/main/resources/application-edge.yaml b/indexer/src/main/resources/application-edge.yaml index c49f45a9..7a1b6803 100644 --- a/indexer/src/main/resources/application-edge.yaml +++ b/indexer/src/main/resources/application-edge.yaml @@ -9,3 +9,7 @@ management: web: exposure: include: "health,info,env,beans,logfile" + +logging: + level: + au.org.aodn.ardcvocabs: DEBUG \ No newline at end of file From a07e148f6f1112acc3cd536dbfac4dce8a50cf94 Mon Sep 17 00:00:00 2001 From: utas-raymondng Date: Tue, 24 Sep 2024 18:07:36 +1000 Subject: [PATCH 2/4] Speed up map processing by using multiple thread --- indexer/src/main/resources/application-edge.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indexer/src/main/resources/application-edge.yaml b/indexer/src/main/resources/application-edge.yaml index 7a1b6803..9c316f38 100644 --- a/indexer/src/main/resources/application-edge.yaml +++ b/indexer/src/main/resources/application-edge.yaml @@ -12,4 +12,4 @@ management: logging: level: - au.org.aodn.ardcvocabs: DEBUG \ No newline at end of file + au.org.aodn.ardcvocabs: DEBUG From d39f36290171b7cca2f3516daf6155659bd790f4 Mon Sep 17 00:00:00 2001 From: utas-raymondng Date: Wed, 25 Sep 2024 11:24:29 +1000 Subject: [PATCH 3/4] Reduce need to new executor --- .../org/aodn/esindexer/configuration/IndexerConfig.java | 7 +++++++ .../java/au/org/aodn/esindexer/utils/GeometryUtils.java | 9 +++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/indexer/src/main/java/au/org/aodn/esindexer/configuration/IndexerConfig.java b/indexer/src/main/java/au/org/aodn/esindexer/configuration/IndexerConfig.java index 9c0af011..19c3033c 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/configuration/IndexerConfig.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/configuration/IndexerConfig.java @@ -14,6 +14,7 @@ import org.springframework.beans.factory.annotation.Value; import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; @Configuration @EnableRetry @@ -26,6 +27,12 @@ public class IndexerConfig { public void init() { GeometryUtils.setCellSize(cellSize); } + + @PreDestroy + public void cleanUp() { + // Clean up resources + GeometryUtils.getExecutorService().shutdown(); + } /** * We need to create component here because we do not want to run test with real http connection * that depends on remote site. The test config need to create an instance of bean for testing diff --git a/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java b/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java index 74c49650..5d1f85d3 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java @@ -32,6 +32,9 @@ public class GeometryUtils { protected static GeometryFactory factory = new GeometryFactory(new PrecisionModel(), 4326); protected static ObjectMapper objectMapper = new ObjectMapper(); protected static Geometry landGeometry; + // Create an ExecutorService with a fixed thread pool size + @Getter + protected static ExecutorService executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); @Getter @Setter @@ -281,9 +284,6 @@ protected static List breakLargeGeometryToGrid(final Geometry large) { // cover Australia List gridPolygons = createGridPolygons(envelope, getCellSize()); - // Create an ExecutorService with a fixed thread pool size - ExecutorService executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); - // List to store Future objects representing the results of the tasks List> futureResults = new ArrayList<>(); @@ -313,9 +313,6 @@ protected static List breakLargeGeometryToGrid(final Geometry large) { } } - // Shutdown the ExecutorService after all tasks are completed - executorService.shutdown(); - logger.debug("End break down large geometry"); return intersectedPolygons; } From 3ff253601c18ecc2dd628fd0cad8db8315e2e323 Mon Sep 17 00:00:00 2001 From: utas-raymondng Date: Wed, 25 Sep 2024 11:28:50 +1000 Subject: [PATCH 4/4] Create new executor on init --- .../au/org/aodn/esindexer/configuration/IndexerConfig.java | 1 + .../main/java/au/org/aodn/esindexer/utils/GeometryUtils.java | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/indexer/src/main/java/au/org/aodn/esindexer/configuration/IndexerConfig.java b/indexer/src/main/java/au/org/aodn/esindexer/configuration/IndexerConfig.java index 19c3033c..c0aea227 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/configuration/IndexerConfig.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/configuration/IndexerConfig.java @@ -26,6 +26,7 @@ public class IndexerConfig { @PostConstruct public void init() { GeometryUtils.setCellSize(cellSize); + GeometryUtils.setExecutorService(Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors())); } @PreDestroy diff --git a/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java b/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java index 5d1f85d3..bb6da16d 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/utils/GeometryUtils.java @@ -34,7 +34,8 @@ public class GeometryUtils { protected static Geometry landGeometry; // Create an ExecutorService with a fixed thread pool size @Getter - protected static ExecutorService executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); + @Setter + protected static ExecutorService executorService; @Getter @Setter