Skip to content

Commit

Permalink
perf: only write actually changed values to config (#207)
Browse files Browse the repository at this point in the history
  • Loading branch information
MikeDombo authored Jun 19, 2023
1 parent b23ade6 commit e18e5cf
Show file tree
Hide file tree
Showing 5 changed files with 319 additions and 282 deletions.
10 changes: 10 additions & 0 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,16 @@
<!-- CloudFront url fronting the device sdk,logging library and component common in S3-->
<url>https://d2jrmugq4soldf.cloudfront.net/snapshots</url>
</repository>
<repository>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
<id>central</id>
<url>https://repo1.maven.org/maven2</url>
</repository>
</repositories>
<dependencyManagement>
<dependencies>
Expand Down
31 changes: 20 additions & 11 deletions src/main/java/com/aws/greengrass/logmanager/LogManagerService.java
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@

import static com.aws.greengrass.componentmanager.KernelConfigResolver.CONFIGURATION_CONFIG_KEY;
import static com.aws.greengrass.logmanager.LogManagerService.LOGS_UPLOADER_SERVICE_TOPICS;
import static com.aws.greengrass.logmanager.util.ConfigUtil.updateFromMapWhenChanged;


@ImplementsService(name = LOGS_UPLOADER_SERVICE_TOPICS, version = "2.0.0")
Expand Down Expand Up @@ -404,7 +405,8 @@ private void loadProcessingFilesConfigDeprecated(String componentName) {
.lookupTopics(PERSISTED_COMPONENT_CURRENT_PROCESSING_FILE_INFORMATION, componentName);


if (currentProcessingComponentTopicsDeprecated != null
if (isDeprecatedVersionSupported()
&& currentProcessingComponentTopicsDeprecated != null
&& !currentProcessingComponentTopicsDeprecated.isEmpty()) {
CurrentProcessingFileInformation processingFileInformation =
CurrentProcessingFileInformation.builder().build();
Expand Down Expand Up @@ -520,15 +522,18 @@ private void handleCloudWatchAttemptStatus(CloudWatchAttempt cloudWatchAttempt)
// Update the runtime configuration and store the last processed file information
context.runOnPublishQueueAndWait(() -> {
processingFilesInformation.forEach((componentName, processingFiles) -> {
// Update old config value to handle downgrade from v 2.3.1 to older ones
Topics componentTopicsDeprecated =
getRuntimeConfig().lookupTopics(PERSISTED_COMPONENT_CURRENT_PROCESSING_FILE_INFORMATION,
componentName);

if (processingFiles.getMostRecentlyUsed() != null) {
componentTopicsDeprecated.updateFromMap(
processingFiles.getMostRecentlyUsed().convertToMapOfObjects(),
new UpdateBehaviorTree(UpdateBehaviorTree.UpdateBehavior.REPLACE, System.currentTimeMillis()));
if (isDeprecatedVersionSupported()) {
// Update old config value to handle downgrade from v 2.3.1 to older ones
Topics componentTopicsDeprecated =
getRuntimeConfig().lookupTopics(PERSISTED_COMPONENT_CURRENT_PROCESSING_FILE_INFORMATION,
componentName);

if (processingFiles.getMostRecentlyUsed() != null) {
updateFromMapWhenChanged(componentTopicsDeprecated,
processingFiles.getMostRecentlyUsed().convertToMapOfObjects(),
new UpdateBehaviorTree(UpdateBehaviorTree.UpdateBehavior.REPLACE,
System.currentTimeMillis()));
}
}

// Handle version 2.3.1 and above
Expand All @@ -537,7 +542,7 @@ private void handleCloudWatchAttemptStatus(CloudWatchAttempt cloudWatchAttempt)
getRuntimeConfig().lookupTopics(PERSISTED_COMPONENT_CURRENT_PROCESSING_FILE_INFORMATION_V2,
componentName);

componentTopics.updateFromMap(processingFiles.toMap(),
updateFromMapWhenChanged(componentTopics, processingFiles.toMap(),
new UpdateBehaviorTree(UpdateBehaviorTree.UpdateBehavior.REPLACE, System.currentTimeMillis()));
});
});
Expand All @@ -552,6 +557,10 @@ private void handleCloudWatchAttemptStatus(CloudWatchAttempt cloudWatchAttempt)
isCurrentlyUploading.set(false);
}

private boolean isDeprecatedVersionSupported() {
return Coerce.toBoolean(getConfig().findOrDefault(true, CONFIGURATION_CONFIG_KEY, "deprecatedVersionSupport"));
}

/**
* Remove the processing files information for a given component from memory and disk (runtime config).
* @param componentName - the name of a component
Expand Down
90 changes: 90 additions & 0 deletions src/main/java/com/aws/greengrass/logmanager/util/ConfigUtil.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
/*
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
* SPDX-License-Identifier: Apache-2.0
*/

package com.aws.greengrass.logmanager.util;

import com.aws.greengrass.config.CaseInsensitiveString;
import com.aws.greengrass.config.Node;
import com.aws.greengrass.config.Topic;
import com.aws.greengrass.config.Topics;
import com.aws.greengrass.config.UnsupportedInputTypeException;
import com.aws.greengrass.config.UpdateBehaviorTree;
import com.aws.greengrass.logging.api.Logger;
import com.aws.greengrass.logging.impl.LogManager;
import edu.umd.cs.findbugs.annotations.NonNull;

import java.util.HashSet;
import java.util.Map;
import java.util.Objects;
import java.util.Set;

public final class ConfigUtil {
private static final Logger logger = LogManager.getLogger(ConfigUtil.class);

private ConfigUtil() {
}

/**
* Same as topics.updateFromMap, but only makes the update when the value actually changes, skipping any unnecessary
* timestampUpdated events. Ideally this code would exist in Topics, but it isn't, so we need to do this in order to
* maintain compatibility.
*
* @param topics Topics to update with values from the map
* @param newValues the new value to apply
* @param ubt update behavior tree
*/
public static void updateFromMapWhenChanged(Topics topics, Map<String, Object> newValues, UpdateBehaviorTree ubt) {
Set<CaseInsensitiveString> childrenToRemove = new HashSet<>(topics.children.keySet());

newValues.forEach((okey, value) -> {
CaseInsensitiveString key = new CaseInsensitiveString(okey);
childrenToRemove.remove(key);
updateChild(topics, key, value, ubt);
});

childrenToRemove.forEach(childName -> {
UpdateBehaviorTree childMergeBehavior = ubt.getChildBehavior(childName.toString());

// remove the existing child if its merge behavior is REPLACE
if (childMergeBehavior.getBehavior() == UpdateBehaviorTree.UpdateBehavior.REPLACE) {
topics.remove(topics.children.get(childName));
}
});
}

private static void updateChild(Topics t, CaseInsensitiveString key, Object value,
@NonNull UpdateBehaviorTree mergeBehavior) {
UpdateBehaviorTree childMergeBehavior = mergeBehavior.getChildBehavior(key.toString());

Node existingChild = t.children.get(key);
// if new node is a container node
if (value instanceof Map) {
// if existing child is a container node
if (existingChild == null || existingChild instanceof Topics) {
updateFromMapWhenChanged(t.createInteriorChild(key.toString()), (Map) value, childMergeBehavior);
} else {
t.remove(existingChild);
Topics newNode = t.createInteriorChild(key.toString(), mergeBehavior.getTimestampToUse());
updateFromMapWhenChanged(newNode, (Map) value, childMergeBehavior);
}
// if new node is a leaf node
} else {
try {
if (existingChild == null || existingChild instanceof Topic) {
Topic node = t.createLeafChild(key.toString());
if (!Objects.equals(node.getOnce(), value)) {
node.withValueChecked(childMergeBehavior.getTimestampToUse(), value);
}
} else {
t.remove(existingChild);
Topic newNode = t.createLeafChild(key.toString());
newNode.withValueChecked(childMergeBehavior.getTimestampToUse(), value);
}
} catch (UnsupportedInputTypeException e) {
logger.error("Should never fail in updateChild", e);
}
}
}
}
Loading

0 comments on commit e18e5cf

Please sign in to comment.