Skip to content

Commit

Permalink
feat: linux control group version 2 API support cgroup v2
Browse files Browse the repository at this point in the history
  • Loading branch information
@ChangxinDong
ChangxinDong committed Oct 21, 2022
1 parent 0ff5dc4 commit 58f4ec1
Show file tree
Hide file tree
Showing 11 changed files with 772 additions and 88 deletions.
3 changes: 2 additions & 1 deletion src/integrationtests/java/com/aws/greengrass/integrationtests/ipc/IPCHibernateTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import com.aws.greengrass.testcommons.testutilities.GGExtension;
import com.aws.greengrass.testcommons.testutilities.TestUtils;
import com.aws.greengrass.util.platforms.unix.linux.Cgroup;
import com.aws.greengrass.util.platforms.unix.linux.CgroupSubSystem;
import com.aws.greengrass.util.platforms.unix.linux.LinuxSystemResourceController;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
Expand Down Expand Up @@ -115,7 +116,7 @@ void GIVEN_LifeCycleEventStreamClient_WHEN_pause_resume_component_THEN_target_se
private LinuxSystemResourceController.CgroupFreezerState getCgroupFreezerState(String serviceName)
throws IOException {
return LinuxSystemResourceController.CgroupFreezerState.valueOf(
new String(Files.readAllBytes(Cgroup.Freezer.getCgroupFreezerStateFilePath(serviceName)),
new String(Files.readAllBytes(new Cgroup(CgroupSubSystem.Freezer).getCgroupFreezerStateFilePath(serviceName)),
StandardCharsets.UTF_8).trim());
}
}
Expand Down
117 changes: 113 additions & 4 deletions ...com/aws/greengrass/integrationtests/lifecyclemanager/GenericExternalServiceIntegTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,11 @@
import com.aws.greengrass.testcommons.testutilities.NoOpPathOwnershipHandler;
import com.aws.greengrass.util.Pair;
import com.aws.greengrass.util.platforms.unix.linux.Cgroup;
import com.aws.greengrass.util.platforms.unix.linux.CgroupSubSystem;
import com.aws.greengrass.util.platforms.unix.linux.CgroupSubSystemV2;
import com.aws.greengrass.util.platforms.unix.linux.LinuxPlatform;
import com.aws.greengrass.util.platforms.unix.linux.LinuxSystemResourceController;
import com.aws.greengrass.util.platforms.unix.linux.LinuxSystemResourceControllerV2;
import org.apache.commons.lang3.SystemUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
Expand All @@ -32,14 +36,18 @@
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import org.mockito.Mock;

import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileSystemException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.CountDownLatch;
Expand Down Expand Up @@ -75,6 +83,7 @@
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assumptions.assumeTrue;
import static org.junit.jupiter.params.provider.Arguments.arguments;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
Expand All @@ -84,6 +93,9 @@ class GenericExternalServiceIntegTest extends BaseITCase {

private Kernel kernel;

@Mock
LinuxPlatform platform;

static Stream<Arguments> posixTestUserConfig() {
return Stream.of(
arguments("config_run_with_user.yaml", "nobody", "nobody"),
Expand Down Expand Up @@ -582,6 +594,87 @@ void GIVEN_linux_resource_limits_WHEN_it_changes_THEN_component_runs_with_new_re
assertResourceLimits(componentName, 10240l * 1024, 1.5);
}

@EnabledOnOs({OS.LINUX})
@Test
void GIVEN_linux_resource_limits_WHEN_it_changes_THEN_component_runs_with_new_resource_limits_V2() throws Exception {
String componentName = "echo_service";
// Run with no resource limit
ConfigPlatformResolver.initKernelWithMultiPlatformConfig(kernel,
getClass().getResource("config_run_with_user.yaml"));
CountDownLatch service = new CountDownLatch(1);
kernel.getContext().addGlobalStateChangeListener((s, oldState, newState) -> {
if (s.getName().equals(componentName) && newState.equals(State.RUNNING)) {
service.countDown();
}
});

String componentPathString = "/sys/fs/cgroup/greengrass/" + componentName;
String cgroupv2RootPathString = "/sys/fs/cgroup";
String cgroupv2GGPathString = "/sys/fs/cgroup/greengrass";
Path cgroupv2GGPath = Paths.get(cgroupv2GGPathString);
Path path = Paths.get(componentPathString);
Path cgroupv2RootPath = Paths.get(cgroupv2RootPathString);

Files.createDirectories(path);

if (!Files.exists(path.resolve("memory.max"))) {
Files.createFile(path.resolve("memory.max"));
}
if (!Files.exists(path.resolve("cpu.max"))) {
Files.createFile(path.resolve("cpu.max"));
}
Files.write(path.resolve("cpu.max"), "max 100000".getBytes(StandardCharsets.UTF_8));

if (!Files.exists(cgroupv2RootPath.resolve("cgroup.subtree_control"))) {
Files.createFile(cgroupv2RootPath.resolve("cgroup.subtree_control"));
}
if (!Files.exists(cgroupv2GGPath.resolve("cgroup.subtree_control"))) {
Files.createFile(cgroupv2GGPath.resolve("cgroup.subtree_control"));
}

kernel.launch();
assertTrue(service.await(20, TimeUnit.SECONDS), "service running");

LinuxSystemResourceControllerV2 linuxSystemResourceControllerV2 = new LinuxSystemResourceControllerV2(platform);
LinuxSystemResourceControllerV2 spyV2 = spy(linuxSystemResourceControllerV2);
Set<String> rootPath = new HashSet<>();
rootPath.add(cgroupv2RootPathString);

lenient().doReturn(spyV2).when(platform).getSystemResourceController();
lenient().doReturn(rootPath).when(spyV2).getMountedPaths();
// Run with nucleus default resource limit
assertResourceLimits_V2(componentName, 10240l * 1024, 1.5);

// Run with component resource limit
kernel.getConfig().lookup(SERVICES_NAMESPACE_TOPIC, componentName, RUN_WITH_NAMESPACE_TOPIC,
SYSTEM_RESOURCE_LIMITS_TOPICS, "memory").withValue(102400l);

kernel.getConfig().lookup(SERVICES_NAMESPACE_TOPIC, componentName, RUN_WITH_NAMESPACE_TOPIC,
SYSTEM_RESOURCE_LIMITS_TOPICS, "cpus").withValue(0.5);
// Block until events are completed
kernel.getContext().waitForPublishQueueToClear();

assertResourceLimits_V2(componentName, 102400l * 1024, 0.5);

// Run with updated component resource limit
kernel.getConfig().lookup(SERVICES_NAMESPACE_TOPIC, componentName, RUN_WITH_NAMESPACE_TOPIC,
SYSTEM_RESOURCE_LIMITS_TOPICS, "memory").withValue(51200l);
kernel.getConfig().lookup(SERVICES_NAMESPACE_TOPIC, componentName, RUN_WITH_NAMESPACE_TOPIC,
SYSTEM_RESOURCE_LIMITS_TOPICS, "cpus").withValue(0.35);
kernel.getConfig().lookup(SERVICES_NAMESPACE_TOPIC, componentName, VERSION_CONFIG_KEY).withValue("2.0.0");
// Block until events are completed
kernel.getContext().waitForPublishQueueToClear();

assertResourceLimits_V2(componentName, 51200l * 1024, 0.35);

//Remove component resource limit, should fall back to default
kernel.getConfig().lookupTopics(SERVICES_NAMESPACE_TOPIC, componentName, RUN_WITH_NAMESPACE_TOPIC,
SYSTEM_RESOURCE_LIMITS_TOPICS).remove();
kernel.getContext().waitForPublishQueueToClear();

assertResourceLimits_V2(componentName, 10240l * 1024, 1.5);
}

@Test
void GIVEN_service_starts_up_WHEN_service_breaks_THEN_status_details_persisted_for_errored_and_broken_states()
throws Exception {
Expand Down Expand Up @@ -713,18 +806,34 @@ void GIVEN_service_starts_up_WHEN_startup_times_out_THEN_timeout_error_code_pers
}

private void assertResourceLimits(String componentName, long memory, double cpus) throws Exception {
byte[] buf1 = Files.readAllBytes(Cgroup.Memory.getComponentMemoryLimitPath(componentName));
byte[] buf1 = Files.readAllBytes(new Cgroup(CgroupSubSystem.Memory).getComponentMemoryLimitPath(componentName));
assertThat(memory, equalTo(Long.parseLong(new String(buf1, StandardCharsets.UTF_8).trim())));

byte[] buf2 = Files.readAllBytes(Cgroup.CPU.getComponentCpuQuotaPath(componentName));
byte[] buf3 = Files.readAllBytes(Cgroup.CPU.getComponentCpuPeriodPath(componentName));
byte[] buf2 = Files.readAllBytes(new Cgroup(CgroupSubSystem.CPU).getComponentCpuQuotaPath(componentName));
byte[] buf3 = Files.readAllBytes(new Cgroup(CgroupSubSystem.CPU).getComponentCpuPeriodPath(componentName));

int quota = Integer.parseInt(new String(buf2, StandardCharsets.UTF_8).trim());
int period = Integer.parseInt(new String(buf3, StandardCharsets.UTF_8).trim());
int expectedQuota = (int) (cpus * period);
assertThat(expectedQuota, equalTo(quota));
}

private void assertResourceLimits_V2(String componentName, long memory, double cpus) throws Exception {
byte[] buf1 = Files.readAllBytes(new Cgroup(CgroupSubSystemV2.Memory).getComponentMemoryLimitPath(componentName));
assertThat(memory, equalTo(Long.parseLong(new String(buf1, StandardCharsets.UTF_8).trim())));

byte[] buf2 = Files.readAllBytes(new Cgroup(CgroupSubSystemV2.CPU).getComponentCpuMaxPath(componentName));

String cpuMaxContent = new String(buf2, StandardCharsets.UTF_8).trim();
String[] cpuMaxContentArr = cpuMaxContent.split(" ");

String cpuMaxStr = cpuMaxContentArr[0];
String cpuPeriodStr = cpuMaxContentArr[1];
int quota = Integer.parseInt(cpuMaxStr);
int expectedQuota = (int) (cpus * Integer.parseInt(cpuPeriodStr));
assertThat(expectedQuota, equalTo(quota));
}

void GIVEN_running_service_WHEN_pause_resume_requested_THEN_pause_resume_Service_and_freeze_thaw_cgroup(
ExtensionContext context) throws Exception {
ignoreExceptionOfType(context, FileSystemException.class);
Expand Down Expand Up @@ -761,7 +870,7 @@ void GIVEN_running_service_WHEN_pause_resume_requested_THEN_pause_resume_Service
private LinuxSystemResourceController.CgroupFreezerState getCgroupFreezerState(String serviceName)
throws IOException {
return LinuxSystemResourceController.CgroupFreezerState
.valueOf(new String(Files.readAllBytes(Cgroup.Freezer.getCgroupFreezerStateFilePath(serviceName))
.valueOf(new String(Files.readAllBytes(new Cgroup(CgroupSubSystem.Freezer).getCgroupFreezerStateFilePath(serviceName))
, StandardCharsets.UTF_8).trim());
}
}
73 changes: 73 additions & 0 deletions src/main/java/com/aws/greengrass/util/platforms/unix/linux/CGroupSubSystemPath.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
/*
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
* SPDX-License-Identifier: Apache-2.0
*/

package com.aws.greengrass.util.platforms.unix.linux;

import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;

import java.nio.file.Path;
import java.nio.file.Paths;

@SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME",
justification = "CGroupSubSystemPath virtual filesystem path cannot be relative")
public interface CGroupSubSystemPath {
String CGROUP_ROOT = "/sys/fs/cgroup";
String GG_NAMESPACE = "greengrass";
String CGROUP_MEMORY_LIMITS = "memory.limit_in_bytes";
String CPU_CFS_PERIOD_US = "cpu.cfs_period_us";
String CPU_CFS_QUOTA_US = "cpu.cfs_quota_us";
String CGROUP_PROCS = "cgroup.procs";
String FREEZER_STATE_FILE = "freezer.state";
String CPU_MAX = "cpu.max";
String MEMORY_MAX = "memory.max";
String CGROUP_SUBTREE_CONTROL = "cgroup.subtree_control";
String CGROUP_FREEZE = "cgroup.freeze";

default Path getRootPath() {
return Paths.get(CGROUP_ROOT);
}

String rootMountCmd();

default String subsystemMountCmd() {
return null;
}

Path getSubsystemRootPath();

Path getSubsystemGGPath();

Path getSubsystemComponentPath(String componentName);

Path getComponentMemoryLimitPath(String componentName);

default Path getComponentCpuPeriodPath(String componentName) {
return null;
}

default Path getComponentCpuQuotaPath(String componentName) {
return null;
}

Path getCgroupProcsPath(String componentName);

Path getCgroupFreezerStateFilePath(String componentName);

default Path getRootSubTreeControlPath() {
return null;
}

default Path getGGSubTreeControlPath() {
return null;
}

default Path getComponentCpuMaxPath(String componentName) {
return null;
}

default Path getCgroupFreezePath(String componentName) {
return null;
}
}
Loading

0 comments on commit 58f4ec1

Please sign in to comment.