Skip to content

Commit

Permalink
feat: linux control group version 2 API support cgroup v2
Browse files Browse the repository at this point in the history
  • Loading branch information
@yiwenTS
yiwenTS committed Oct 25, 2022
1 parent 52800bd commit 1abc7cc
Show file tree
Hide file tree
Showing 7 changed files with 93 additions and 82 deletions.
147 changes: 81 additions & 66 deletions ...com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,36 +18,35 @@
import com.aws.greengrass.logging.impl.GreengrassLogMessage;
import com.aws.greengrass.logging.impl.Slf4jLogAdapter;
import com.aws.greengrass.status.model.ComponentStatusDetails;
import com.aws.greengrass.testcommons.testutilities.GGExtension;
import com.aws.greengrass.testcommons.testutilities.NoOpPathOwnershipHandler;
import com.aws.greengrass.util.Pair;
import com.aws.greengrass.util.platforms.unix.linux.Cgroup;
import com.aws.greengrass.util.platforms.unix.linux.CgroupSubSystem;
import com.aws.greengrass.util.platforms.unix.linux.CgroupSubSystemV2;
import com.aws.greengrass.util.platforms.unix.linux.LinuxPlatform;
import com.aws.greengrass.util.platforms.unix.linux.LinuxSystemResourceController;
import com.aws.greengrass.util.platforms.unix.linux.LinuxSystemResourceControllerV2;
import com.aws.greengrass.util.platforms.SystemResourceController;
import com.aws.greengrass.util.platforms.unix.linux.*;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.SystemUtils;
import org.apache.commons.lang3.reflect.FieldUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledOnOs;
import org.junit.jupiter.api.condition.OS;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.extension.ExtensionContext;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import org.mockito.Mock;
import org.mockito.*;
import org.mockito.junit.jupiter.MockitoExtension;

import java.io.IOException;
import java.lang.reflect.Field;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileSystemException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.CountDownLatch;
Expand Down Expand Up @@ -83,18 +82,22 @@
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assumptions.assumeTrue;
import static org.junit.jupiter.params.provider.Arguments.arguments;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;


@ExtendWith({GGExtension.class, MockitoExtension.class})
class GenericExternalServiceIntegTest extends BaseITCase {

private Kernel kernel;

@Mock
LinuxPlatform platform;
LinuxPlatform linuxPlatform;

private final static String ROOT_PATH_STRING = "/systest21/fs/cgroup";
private final static String GG_PATH_STRING = "greengrass";

@Spy
SystemResourceController spySystemResourceController;

static Stream<Arguments> posixTestUserConfig() {
return Stream.of(
Expand Down Expand Up @@ -635,82 +638,78 @@ void GIVEN_linux_resource_limits_WHEN_it_changes_THEN_component_runs_with_new_re
@EnabledOnOs({OS.LINUX})
@Test
void GIVEN_linux_resource_limits_WHEN_it_changes_THEN_component_runs_with_new_resource_limits_V2() throws Exception {
String componentName = "echo_service";
String echoComponentName = "echo_service";
String mainComponentName = "main";
String rootGGPathString = ROOT_PATH_STRING + "/" + GG_PATH_STRING;
String componentPathString = rootGGPathString + "/" + echoComponentName;
// Run with no resource limit
ConfigPlatformResolver.initKernelWithMultiPlatformConfig(kernel,
getClass().getResource("config_run_with_user.yaml"));
CountDownLatch service = new CountDownLatch(1);
kernel.getContext().addGlobalStateChangeListener((s, oldState, newState) -> {
if (s.getName().equals(componentName) && newState.equals(State.RUNNING)) {
if (s.getName().equals(echoComponentName) && newState.equals(State.RUNNING)) {
service.countDown();
}
});

String componentPathString = "/sys/fs/cgroup/greengrass/" + componentName;
String cgroupv2RootPathString = "/sys/fs/cgroup";
String cgroupv2GGPathString = "/sys/fs/cgroup/greengrass";
Path cgroupv2GGPath = Paths.get(cgroupv2GGPathString);
Path path = Paths.get(componentPathString);
Path cgroupv2RootPath = Paths.get(cgroupv2RootPathString);

Files.createDirectories(path);

if (!Files.exists(path.resolve("memory.max"))) {
Files.createFile(path.resolve("memory.max"));
}
if (!Files.exists(path.resolve("cpu.max"))) {
Files.createFile(path.resolve("cpu.max"));
}
Files.write(path.resolve("cpu.max"), "max 100000".getBytes(StandardCharsets.UTF_8));
linuxPlatform = kernel.getContext().get(LinuxPlatform.class);

if (!Files.exists(cgroupv2RootPath.resolve("cgroup.subtree_control"))) {
Files.createFile(cgroupv2RootPath.resolve("cgroup.subtree_control"));
}
if (!Files.exists(cgroupv2GGPath.resolve("cgroup.subtree_control"))) {
Files.createFile(cgroupv2GGPath.resolve("cgroup.subtree_control"));
}
createComponentData(echoComponentName);
createComponentData(mainComponentName);

kernel.launch();
assertTrue(service.await(20, TimeUnit.SECONDS), "service running");
setFinalStatic(linuxPlatform, LinuxPlatform.class.getDeclaredField("CGROUP_CONTROLLERS"), Paths.get(componentPathString + "/memory.max"));

LinuxSystemResourceControllerV2 linuxSystemResourceControllerV2 = new LinuxSystemResourceControllerV2(platform);
LinuxSystemResourceControllerV2 spyV2 = spy(linuxSystemResourceControllerV2);
Set<String> rootPath = new HashSet<>();
rootPath.add(cgroupv2RootPathString);
spySystemResourceController = linuxPlatform.getSystemResourceController();
LinuxSystemResourceControllerV2 controllerV2 = (LinuxSystemResourceControllerV2) spySystemResourceController;

lenient().doReturn(spyV2).when(platform).getSystemResourceController();
lenient().doReturn(rootPath).when(spyV2).getMountedPaths();
// Run with nucleus default resource limit
assertResourceLimits_V2(componentName, 10240l * 1024, 1.5);
Field memoryCgroupField = LinuxSystemResourceControllerV2.class.getSuperclass().getDeclaredField("memoryCgroup");
memoryCgroupField.setAccessible(true);
Cgroup memoryCgroup = (Cgroup) memoryCgroupField.get(controllerV2);
Field subsystem = memoryCgroup.getClass().getDeclaredField("subSystem");
subsystem.setAccessible(true);
CgroupSubSystemV2 cg = (CgroupSubSystemV2) subsystem.get(memoryCgroup);
Field f = cg.getClass().getInterfaces()[0].getDeclaredField("CGROUP_ROOT");
setFinalStatic(cg, f, Paths.get(ROOT_PATH_STRING));

// Run with component resource limit
kernel.getConfig().lookup(SERVICES_NAMESPACE_TOPIC, componentName, RUN_WITH_NAMESPACE_TOPIC,
SYSTEM_RESOURCE_LIMITS_TOPICS, "memory").withValue(102400l);
Field mountsField = LinuxSystemResourceControllerV2.class.getSuperclass().getDeclaredField("MOUNT_PATH");
mountsField.setAccessible(true);
String mountPathFile = rootGGPathString + "/mountPath.txt";
final Path mountPathFilePath = Paths.get(mountPathFile);
if (!Files.exists(mountPathFilePath)) {
Files.createFile(mountPathFilePath);
Files.write(mountPathFilePath, String.format("test1 %s test2 test3 test4 test5", ROOT_PATH_STRING).getBytes(StandardCharsets.UTF_8));

kernel.getConfig().lookup(SERVICES_NAMESPACE_TOPIC, componentName, RUN_WITH_NAMESPACE_TOPIC,
SYSTEM_RESOURCE_LIMITS_TOPICS, "cpus").withValue(0.5);
// Block until events are completed
kernel.getContext().waitForPublishQueueToClear();
}
setFinalStatic(controllerV2, mountsField, mountPathFile);

assertResourceLimits_V2(componentName, 102400l * 1024, 0.5);
kernel.launch();
assertResourceLimits_V2(10240l * 1024, 1.5);

// Run with updated component resource limit
kernel.getConfig().lookup(SERVICES_NAMESPACE_TOPIC, componentName, RUN_WITH_NAMESPACE_TOPIC,
kernel.getConfig().lookup(SERVICES_NAMESPACE_TOPIC, echoComponentName, RUN_WITH_NAMESPACE_TOPIC,
SYSTEM_RESOURCE_LIMITS_TOPICS, "memory").withValue(51200l);
kernel.getConfig().lookup(SERVICES_NAMESPACE_TOPIC, componentName, RUN_WITH_NAMESPACE_TOPIC,
kernel.getConfig().lookup(SERVICES_NAMESPACE_TOPIC, echoComponentName, RUN_WITH_NAMESPACE_TOPIC,
SYSTEM_RESOURCE_LIMITS_TOPICS, "cpus").withValue(0.35);
kernel.getConfig().lookup(SERVICES_NAMESPACE_TOPIC, componentName, VERSION_CONFIG_KEY).withValue("2.0.0");
kernel.getConfig().lookup(SERVICES_NAMESPACE_TOPIC, echoComponentName, VERSION_CONFIG_KEY).withValue("2.0.0");
// Block until events are completed
kernel.getContext().waitForPublishQueueToClear();

assertResourceLimits_V2(componentName, 51200l * 1024, 0.35);
assertResourceLimits_V2(51200l * 1024, 0.35);

//Remove component resource limit, should fall back to default
kernel.getConfig().lookupTopics(SERVICES_NAMESPACE_TOPIC, componentName, RUN_WITH_NAMESPACE_TOPIC,
kernel.getConfig().lookupTopics(SERVICES_NAMESPACE_TOPIC, echoComponentName, RUN_WITH_NAMESPACE_TOPIC,
SYSTEM_RESOURCE_LIMITS_TOPICS).remove();
kernel.getContext().waitForPublishQueueToClear();

assertResourceLimits_V2(componentName, 10240l * 1024, 1.5);
assertResourceLimits_V2(10240l * 1024, 1.5);

FileUtils.deleteDirectory(Paths.get("/systest21").toFile());
}

private void setFinalStatic(Object obj, Field field, Object newValue) throws Exception {
field.setAccessible(true);
FieldUtils.removeFinalModifier(field, true);
field.set(obj, newValue);
}

@Test
Expand Down Expand Up @@ -856,11 +855,27 @@ private void assertResourceLimits(String componentName, long memory, double cpus
assertThat(expectedQuota, equalTo(quota));
}

private void assertResourceLimits_V2(String componentName, long memory, double cpus) throws Exception {
byte[] buf1 = Files.readAllBytes(new Cgroup(CgroupSubSystemV2.Memory).getComponentMemoryLimitPath(componentName));
private void createComponentData(String componentName) throws IOException {
Path path = Paths.get(ROOT_PATH_STRING).resolve(GG_PATH_STRING).resolve(componentName);
Files.createDirectories(path);

if (!Files.exists(path.resolve("memory.max"))) {
Files.createFile(path.resolve("memory.max"));
}
if (!Files.exists(path.resolve("cpu.max"))) {
Files.createFile(path.resolve("cpu.max"));
}
Files.write(path.resolve("cpu.max"), "max 100000".getBytes(StandardCharsets.UTF_8));
if (!Files.exists(path.resolve("cgroup.proc"))) {
Files.createFile(path.resolve("cgroup.proc"));
}
}

private void assertResourceLimits_V2(long memory, double cpus) throws Exception {
byte[] buf1 = Files.readAllBytes(Paths.get(String.format("%s/%s/echo_service/memory.max", ROOT_PATH_STRING, GG_PATH_STRING)));
assertThat(memory, equalTo(Long.parseLong(new String(buf1, StandardCharsets.UTF_8).trim())));

byte[] buf2 = Files.readAllBytes(new Cgroup(CgroupSubSystemV2.CPU).getComponentCpuMaxPath(componentName));
byte[] buf2 = Files.readAllBytes(Paths.get(String.format("%s/%s/echo_service/cpu.max", ROOT_PATH_STRING, GG_PATH_STRING)));

String cpuMaxContent = new String(buf2, StandardCharsets.UTF_8).trim();
String[] cpuMaxContentArr = cpuMaxContent.split(" ");
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupSubSystemPath.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
@SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME",
justification = "CGroupSubSystemPath virtual filesystem path cannot be relative")
public interface CGroupSubSystemPath {
String CGROUP_ROOT = "/sys/fs/cgroup";
Path CGROUP_ROOT = Paths.get("/sys/fs/cgroup");
String GG_NAMESPACE = "greengrass";
String CGROUP_MEMORY_LIMITS = "memory.limit_in_bytes";
String CPU_CFS_PERIOD_US = "cpu.cfs_period_us";
Expand All @@ -26,7 +26,7 @@ public interface CGroupSubSystemPath {
String CGROUP_FREEZE = "cgroup.freeze";

default Path getRootPath() {
return Paths.get(CGROUP_ROOT);
return CGROUP_ROOT;
}

String rootMountCmd();
Expand Down
3 changes: 1 addition & 2 deletions src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystem.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;

import java.nio.file.Path;
import java.nio.file.Paths;

@SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME",
justification = "CgroupSubSystem virtual filesystem path cannot be relative")
Expand Down Expand Up @@ -53,7 +52,7 @@ public String subsystemMountCmd() {

@Override
public Path getSubsystemRootPath() {
return Paths.get(CGROUP_ROOT).resolve(osString);
return CGROUP_ROOT.resolve(osString);
}

@Override
Expand Down
3 changes: 1 addition & 2 deletions src/main/java/com/aws/greengrass/util/platforms/unix/linux/CgroupSubSystemV2.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;

import java.nio.file.Path;
import java.nio.file.Paths;

@SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME",
justification = "CgroupSubSystemV2 virtual filesystem path cannot be relative")
Expand All @@ -22,7 +21,7 @@ public String rootMountCmd() {

@Override
public Path getSubsystemRootPath() {
return Paths.get(CGROUP_ROOT);
return CGROUP_ROOT;
}

@Override
Expand Down
8 changes: 2 additions & 6 deletions src/main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxPlatform.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,14 @@
@SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME",
justification = "Cgroup Controller virtual filesystem path cannot be relative")
public class LinuxPlatform extends UnixPlatform {
private static final String CGROUP_ROOT = "/sys/fs/cgroup";
private static final String CGROUP_CONTROLLERS = "cgroup.controllers";
private static final Path CGROUP_CONTROLLERS = Paths.get("/sys/fs/cgroup/cgroup.controllers");

SystemResourceController systemResourceController;

@Override
public SystemResourceController getSystemResourceController() {
//if the path exists, identify it as cgroupv1, otherwise identify it as cgroupv2
if (Files.exists(getControllersRootPath())) {
if (Files.exists(CGROUP_CONTROLLERS)) {
systemResourceController = new LinuxSystemResourceControllerV2(this);
} else {
systemResourceController = new LinuxSystemResourceController(this);
Expand All @@ -33,7 +32,4 @@ public SystemResourceController getSystemResourceController() {
return systemResourceController;
}

private Path getControllersRootPath() {
return Paths.get(CGROUP_ROOT).resolve(CGROUP_CONTROLLERS);
}
}
7 changes: 5 additions & 2 deletions ...main/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ public class LinuxSystemResourceController implements SystemResourceController {
protected static final String MEMORY_KEY = "memory";
protected static final String CPUS_KEY = "cpus";
private static final String UNICODE_SPACE = "\\040";
protected static final String MOUNT_PATH = "/proc/self/mounts";
protected Cgroup memoryCgroup;
protected Cgroup cpuCgroup;
protected Cgroup freezerCgroup;
Expand All @@ -48,6 +49,8 @@ public class LinuxSystemResourceController implements SystemResourceController {

protected LinuxPlatform platform;

protected Set<String> mounts;

public LinuxSystemResourceController() {

}
Expand Down Expand Up @@ -243,7 +246,7 @@ private void handleErrorAddingPidToCgroup(IOException e, String component) {
public Set<String> getMountedPaths() throws IOException {
Set<String> mountedPaths = new HashSet<>();

Path procMountsPath = Paths.get("/proc/self/mounts");
Path procMountsPath = Paths.get(MOUNT_PATH);
List<String> mounts = Files.readAllLines(procMountsPath);
for (String mount : mounts) {
String[] split = mount.split(" ");
Expand All @@ -266,7 +269,7 @@ public Set<String> getMountedPaths() throws IOException {
}

protected void initializeCgroup(GreengrassService component, Cgroup cgroup) throws IOException {
Set<String> mounts = getMountedPaths();
mounts = getMountedPaths();

if (!mounts.contains(cgroup.getRootPath().toString())) {
platform.runCmd(cgroup.rootMountCmd(), o -> {
Expand Down
3 changes: 1 addition & 2 deletions ...in/java/com/aws/greengrass/util/platforms/unix/linux/LinuxSystemResourceControllerV2.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Set;

/**
* https://www.kernel.org/doc/Documentation/cgroup-v2.txt
Expand Down Expand Up @@ -81,7 +80,7 @@ public void resumeComponentProcesses(GreengrassService component) throws IOExcep

@Override
protected void initializeCgroup(GreengrassService component, Cgroup cgroup) throws IOException {
Set<String> mounts = getMountedPaths();
mounts = getMountedPaths();

if (!mounts.contains(cgroup.getRootPath().toString())) {
platform.runCmd(cgroup.rootMountCmd(), o -> {
Expand Down

0 comments on commit 1abc7cc

Please sign in to comment.